From df3ef08ff9e3f2b58246b1f52644f4728ece6717 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 2 Apr 2026 08:08:28 +0000 Subject: [PATCH 001/110] 1st test push interface --- internal/apiserver/apiserver.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index f31abaa..8c4cbf9 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -2,6 +2,7 @@ package apiserver import ( "encoding/json" + "log" "net/http" "github.com/gnmic/operator/internal/controller" @@ -27,6 +28,7 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServe func (a *APIServer) routes(mux *http.ServeMux) { mux.HandleFunc("GET /clusters/{namespace}/{name}/plan", a.getClusterPlan) + mux.HandleFunc("POST /clusters/{namespace}/{name}/createTarget", a.postCreateTarget) } func (a *APIServer) getClusterPlan(w http.ResponseWriter, r *http.Request) { @@ -43,3 +45,11 @@ func (a *APIServer) getClusterPlan(w http.ResponseWriter, r *http.Request) { return } } + +// kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 +// curl -X POST http://localhost:8082/clusters/gnmic-system/gnmic-controller-manager/createTarget + +func (a *APIServer) postCreateTarget(w http.ResponseWriter, r *http.Request) { + log.Printf("received POST request: path=%s method=%s remote=%s", r.URL.Path, r.Method, r.RemoteAddr) + w.WriteHeader(http.StatusAccepted) +} From fe8dc49a025c00313d67916129917d23673c7e95 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 2 Apr 2026 14:14:35 +0000 Subject: [PATCH 002/110] change to gin-gonic --- cmd/main.go | 48 +++++++-------- go.mod | 46 +++++++++++---- go.sum | 100 +++++++++++++++++++++++++------- internal/apiserver/apiserver.go | 42 +++++--------- 4 files changed, 155 insertions(+), 81 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 4c37a0d..d9758fa 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -17,10 +17,8 @@ limitations under the License. package main import ( - "context" "flag" "os" - "time" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -33,7 +31,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/manager" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" @@ -219,26 +216,31 @@ func main() { } if apiAddr != "" { - apiServer := apiserver.New(apiAddr, clusterReconciler) - err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - errCh := make(chan error) - go func() { - errCh <- apiServer.Server.ListenAndServe() - }() - select { - case err := <-errCh: - return err - case <-ctx.Done(): - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - return apiServer.Server.Shutdown(ctx) - } - })) - if err != nil { - setupLog.Error(err, "unable to add api server") - os.Exit(1) - } - } + apiserver.New(apiAddr, clusterReconciler) + + } + + // if apiAddr != "" { + // apiServer := apiserver.New(apiAddr, clusterReconciler) + // err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + // errCh := make(chan error) + // go func() { + // errCh <- apiServer.Server.ListenAndServe() + // }() + // select { + // case err := <-errCh: + // return err + // case <-ctx.Done(): + // ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + // defer cancel() + // return apiServer.Server.Shutdown(ctx) + // } + // })) + // if err != nil { + // setupLog.Error(err, "unable to add api server") + // os.Exit(1) + // } + // } // start manager setupLog.Info("starting manager") diff --git a/go.mod b/go.mod index f236ded..208f424 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.25.5 require ( github.com/cert-manager/cert-manager v1.19.3 github.com/go-logr/logr v1.4.3 - github.com/onsi/ginkgo/v2 v2.27.3 - github.com/onsi/gomega v1.38.3 + github.com/onsi/ginkgo/v2 v2.28.1 + github.com/onsi/gomega v1.39.0 github.com/openconfig/gnmic/pkg/api v0.1.10 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.35.1 @@ -17,6 +17,31 @@ require ( sigs.k8s.io/controller-runtime v0.22.4 ) +require ( + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/gabriel-vasile/mimetype v1.4.12 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.30.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect + go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + golang.org/x/arch v0.22.0 // indirect + golang.org/x/crypto v0.48.0 // indirect +) + require ( cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect @@ -27,6 +52,7 @@ require ( github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/gin-gonic/gin v1.12.0 github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.22.4 // indirect github.com/go-openapi/jsonreference v0.21.4 // indirect @@ -46,7 +72,7 @@ require ( github.com/google/btree v1.1.3 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20251213031049-b05bdaca462f // indirect + github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect github.com/google/uuid v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -63,15 +89,15 @@ require ( go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/mod v0.31.0 // indirect - golang.org/x/net v0.48.0 // indirect + golang.org/x/mod v0.32.0 // indirect + golang.org/x/net v0.51.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.39.0 // indirect - golang.org/x/term v0.38.0 // indirect - golang.org/x/text v0.32.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/term v0.40.0 // indirect + golang.org/x/text v0.34.0 // indirect golang.org/x/time v0.14.0 // indirect - golang.org/x/tools v0.40.0 // indirect + golang.org/x/tools v0.41.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect google.golang.org/grpc v1.79.3 // indirect @@ -83,6 +109,6 @@ require ( sigs.k8s.io/gateway-api v1.4.1 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.1 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index 8a613b4..8b1369f 100644 --- a/go.sum +++ b/go.sum @@ -4,10 +4,18 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/cert-manager/cert-manager v1.19.3 h1:3d0Nk/HO3BOmAdBJNaBh+6YgaO3Ciey3xCpOjiX5Obs= github.com/cert-manager/cert-manager v1.19.3/go.mod h1:e9NzLtOKxTw7y99qLyWGmPo6mrC1Nh0EKKcMkRfK+GE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -22,6 +30,12 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= +github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= +github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= @@ -68,10 +82,20 @@ github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxE github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= -github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= @@ -83,8 +107,8 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20251213031049-b05bdaca462f h1:HU1RgM6NALf/KW9HEY6zry3ADbDKcmpQ+hJedoNGQYQ= -github.com/google/pprof v0.0.0-20251213031049-b05bdaca462f/go.mod h1:67FPmZWbr+KDT/VlpWtw6sO9XSjpJmLuHpoLmWiTGgY= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= @@ -93,14 +117,20 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -111,12 +141,14 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.27.3 h1:ICsZJ8JoYafeXFFlFAG75a7CxMsJHwgKwtO+82SE9L8= -github.com/onsi/ginkgo/v2 v2.27.3/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= -github.com/onsi/gomega v1.38.3 h1:eTX+W6dobAYfFeGC2PV6RwXRu/MyT+cQguijutvkpSM= -github.com/onsi/gomega v1.38.3/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= +github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= +github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= +github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q= +github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= github.com/openconfig/gnmic/pkg/api v0.1.10 h1:zU57bogHrnraDFCYDnxHZB8Hcd53bWx1fDkRTPw/R2w= github.com/openconfig/gnmic/pkg/api v0.1.10/go.mod h1:6PntONfjCMq3XzsDfWMkLeoVuBRbkm2foQO5m6PeYo0= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -130,14 +162,24 @@ github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+L github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= @@ -148,8 +190,14 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= @@ -164,6 +212,8 @@ go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6 go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= @@ -172,24 +222,29 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= -golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= +golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= -golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= -golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= -golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= +golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= @@ -209,6 +264,7 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.35.1 h1:0PO/1FhlK/EQNVK5+txc4FuhQibV25VLSdLMmGpDE/Q= @@ -233,7 +289,7 @@ sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5E sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.1 h1:JrhdFMqOd/+3ByqlP2I45kTOZmTRLBUm5pvRjeheg7E= -sigs.k8s.io/structured-merge-diff/v6 v6.3.1/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 h1:2WOzJpHUBVrrkDjU4KBT8n5LDcj824eX0I5UKcgeRUs= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 8c4cbf9..7370be9 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -1,55 +1,45 @@ package apiserver import ( - "encoding/json" "log" - "net/http" + "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" ) type APIServer struct { - Server *http.Server + Server *gin.Engine clusterReconciler *controller.ClusterReconciler } func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer { - mux := http.NewServeMux() + _ = addr a := &APIServer{ - Server: &http.Server{ - Addr: addr, - Handler: mux, - }, + Server: gin.Default(), clusterReconciler: clusterReconciler, } - a.routes(mux) - return a -} -func (a *APIServer) routes(mux *http.ServeMux) { - mux.HandleFunc("GET /clusters/{namespace}/{name}/plan", a.getClusterPlan) - mux.HandleFunc("POST /clusters/{namespace}/{name}/createTarget", a.postCreateTarget) + a.Server.POST("/clusters/:namespace/:name/createTarget", a.postCreateTarget) + a.Server.GET("/clusters/:namespace/:name/plan", a.getClusterPlan) + a.Server.Run(":8082") + return a } -func (a *APIServer) getClusterPlan(w http.ResponseWriter, r *http.Request) { - namespace, name := r.PathValue("namespace"), r.PathValue("name") +func (a *APIServer) getClusterPlan(c *gin.Context) { + log.Printf("received GET request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) + namespace, name := c.Param("namespace"), c.Param("name") plan, err := a.clusterReconciler.GetClusterPlan(namespace, name) if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - w.Header().Set("Content-Type", "application/json") - err = json.NewEncoder(w).Encode(plan) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + c.String(404, err.Error()) return } + c.JSON(200, plan) } // kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 // curl -X POST http://localhost:8082/clusters/gnmic-system/gnmic-controller-manager/createTarget -func (a *APIServer) postCreateTarget(w http.ResponseWriter, r *http.Request) { - log.Printf("received POST request: path=%s method=%s remote=%s", r.URL.Path, r.Method, r.RemoteAddr) - w.WriteHeader(http.StatusAccepted) +func (a *APIServer) postCreateTarget(c *gin.Context) { + log.Printf("received POST request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) + c.Status(202) } From 5e9aedc8e90d411ea220efede79413e53a7815b4 Mon Sep 17 00:00:00 2001 From: Janooski Date: Tue, 7 Apr 2026 20:08:55 +0000 Subject: [PATCH 003/110] change to gin --- cmd/main.go | 70 ++++++++++++------------ config/default/manager_config_patch.yaml | 10 +++- internal/apiserver/apiserver.go | 27 +++++++-- 3 files changed, 64 insertions(+), 43 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index d9758fa..496e4a7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -17,8 +17,12 @@ limitations under the License. package main import ( + "context" + "errors" "flag" + "net/http" "os" + "time" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -29,8 +33,8 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" @@ -64,7 +68,7 @@ func main() { flag.StringVar(&apiAddr, "api-bind-address", "", "The address the operator API endpoint binds to. Disabled if empty.") flag.BoolVar(&devMode, "dev-mode", false, "Enable development mode.") flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", "0", "Disabled: health and readiness probes are served by the gin API server.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") @@ -206,41 +210,35 @@ func main() { } //+kubebuilder:scaffold:builder - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") - os.Exit(1) - } - if apiAddr != "" { - apiserver.New(apiAddr, clusterReconciler) - - } - - // if apiAddr != "" { - // apiServer := apiserver.New(apiAddr, clusterReconciler) - // err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - // errCh := make(chan error) - // go func() { - // errCh <- apiServer.Server.ListenAndServe() - // }() - // select { - // case err := <-errCh: - // return err - // case <-ctx.Done(): - // ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - // defer cancel() - // return apiServer.Server.Shutdown(ctx) - // } - // })) - // if err != nil { - // setupLog.Error(err, "unable to add api server") - // os.Exit(1) - // } - // } + apiServer := apiserver.New(apiAddr, clusterReconciler) + err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + errCh := make(chan error, 1) + go func() { + err := apiServer.Server.ListenAndServe() + if err != nil && !errors.Is(err, http.ErrServerClosed) { + errCh <- err + } + close(errCh) + }() + + select { + case err, ok := <-errCh: + if !ok { + return nil + } + return err + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return apiServer.Server.Shutdown(shutdownCtx) + } + })) + if err != nil { + setupLog.Error(err, "unable to add api server") + os.Exit(1) + } + } // start manager setupLog.Info("starting manager") diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml index 691d2c3..d88bad0 100644 --- a/config/default/manager_config_patch.yaml +++ b/config/default/manager_config_patch.yaml @@ -9,10 +9,18 @@ spec: containers: - name: manager args: - - "--health-probe-bind-address=:8081" + - "--health-probe-bind-address=0" - "--metrics-bind-address=:8080" - "--api-bind-address=:8082" - "--leader-elect" + livenessProbe: + httpGet: + path: /healthz + port: 8082 + readinessProbe: + httpGet: + path: /readyz + port: 8082 ports: - containerPort: 8080 protocol: TCP diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 7370be9..5ffa29d 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -2,29 +2,44 @@ package apiserver import ( "log" + "net/http" "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" ) type APIServer struct { - Server *gin.Engine + Server *http.Server + router *gin.Engine clusterReconciler *controller.ClusterReconciler } func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer { - _ = addr + router := gin.Default() a := &APIServer{ - Server: gin.Default(), + Server: &http.Server{ + Addr: addr, + Handler: router, + }, + router: router, clusterReconciler: clusterReconciler, } - a.Server.POST("/clusters/:namespace/:name/createTarget", a.postCreateTarget) - a.Server.GET("/clusters/:namespace/:name/plan", a.getClusterPlan) - a.Server.Run(":8082") + a.router.POST("/clusters/:namespace/:name/createTarget", a.postCreateTarget) + a.router.GET("/clusters/:namespace/:name/plan", a.getClusterPlan) + a.router.GET("/healthz", a.getHealthz) + a.router.GET("/readyz", a.getReadyz) return a } +func (a *APIServer) getHealthz(c *gin.Context) { + c.Status(http.StatusOK) +} + +func (a *APIServer) getReadyz(c *gin.Context) { + c.Status(http.StatusOK) +} + func (a *APIServer) getClusterPlan(c *gin.Context) { log.Printf("received GET request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) namespace, name := c.Param("namespace"), c.Param("name") From b7e12d7b6e9c958bf523614c931a6c7fe7f1acd8 Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 8 Apr 2026 18:42:37 +0000 Subject: [PATCH 004/110] adding openapi.yaml and generated code, still has bugs --- cmd/main.go | 8 +- go.mod | 16 +++ go.sum | 125 ++++++++++++++++++++++ internal/apiserver/apiserver.go | 29 +++-- internal/apiserver/cfg.yaml | 6 ++ internal/apiserver/gen.go | 184 ++++++++++++++++++++++++++++++++ internal/apiserver/openapi.yaml | 45 ++++++++ 7 files changed, 399 insertions(+), 14 deletions(-) create mode 100644 internal/apiserver/cfg.yaml create mode 100644 internal/apiserver/gen.go create mode 100644 internal/apiserver/openapi.yaml diff --git a/cmd/main.go b/cmd/main.go index 496e4a7..ba4c3a3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -211,11 +211,13 @@ func main() { //+kubebuilder:scaffold:builder if apiAddr != "" { - apiServer := apiserver.New(apiAddr, clusterReconciler) + server, r := apiserver.New(apiAddr, clusterReconciler) + apiserver.RegisterHandlers(r, server) + err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error, 1) go func() { - err := apiServer.Server.ListenAndServe() + err := server.Server.ListenAndServe() if err != nil && !errors.Is(err, http.ErrServerClosed) { errCh <- err } @@ -231,7 +233,7 @@ func main() { case <-ctx.Done(): shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - return apiServer.Server.Shutdown(shutdownCtx) + return server.Server.Shutdown(shutdownCtx) } })) if err != nil { diff --git a/go.mod b/go.mod index 208f424..36c110d 100644 --- a/go.mod +++ b/go.mod @@ -22,24 +22,38 @@ require ( github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect + github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect github.com/gabriel-vasile/mimetype v1.4.12 // indirect + github.com/getkin/kin-openapi v0.133.0 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.30.1 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/goccy/go-yaml v1.19.2 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect + github.com/mailru/easyjson v0.9.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/oapi-codegen/oapi-codegen/v2 v2.6.0 // indirect + github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect + github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.0 // indirect + github.com/speakeasy-api/jsonpath v0.6.0 // indirect + github.com/speakeasy-api/openapi-overlay v0.10.2 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect + github.com/vmware-labs/yaml-jsonpath v0.3.2 // indirect + github.com/woodsbury/decimal128 v1.3.0 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect golang.org/x/arch v0.22.0 // indirect golang.org/x/crypto v0.48.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -112,3 +126,5 @@ require ( sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) + +tool github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen diff --git a/go.sum b/go.sum index 8b1369f..c9059d3 100644 --- a/go.sum +++ b/go.sum @@ -14,24 +14,34 @@ github.com/cert-manager/cert-manager v1.19.3 h1:3d0Nk/HO3BOmAdBJNaBh+6YgaO3Ciey3 github.com/cert-manager/cert-manager v1.19.3/go.mod h1:e9NzLtOKxTw7y99qLyWGmPo6mrC1Nh0EKKcMkRfK+GE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960/go.mod h1:9HQzr9D/0PGwMEbC3d5AB7oi67+h4TsQqItC1GVYG58= +github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 h1:PRxIJD8XjimM5aTknUK9w6DHLDox2r2M3DI4i2pnd3w= +github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936/go.mod h1:ttYvX5qlB+mlV1okblJqcSMtR4c52UKxDiX9GRBS8+Q= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/getkin/kin-openapi v0.133.0 h1:pJdmNohVIJ97r4AUFtEXRXwESr8b0bD721u/Tz6k8PQ= +github.com/getkin/kin-openapi v0.133.0/go.mod h1:boAciF6cXk5FhPqe/NQeBTeenbjqU4LhWBf09ILVvWE= github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= @@ -90,27 +100,46 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -119,14 +148,19 @@ github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zt github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -139,16 +173,38 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oapi-codegen/oapi-codegen/v2 v2.6.0 h1:4i+F2cvwBFZeplxCssNdLy3MhNzUD87mI3HnayHZkAU= +github.com/oapi-codegen/oapi-codegen/v2 v2.6.0/go.mod h1:eWHeJSohQJIINJZzzQriVynfGsnlQVh0UkN2UYYcw4Q= +github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//JalHPu/3yz+De2J+4aLtSRlHiY= +github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037/go.mod h1:2bpvgLBZEtENV5scfDFEtB/5+1M4hkQhDQrccEJ/qGw= +github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 h1:bQx3WeLcUWy+RletIKwUIt4x3t8n2SxavmoclizMb8c= +github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q= github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= github.com/openconfig/gnmic/pkg/api v0.1.10 h1:zU57bogHrnraDFCYDnxHZB8Hcd53bWx1fDkRTPw/R2w= github.com/openconfig/gnmic/pkg/api v0.1.10/go.mod h1:6PntONfjCMq3XzsDfWMkLeoVuBRbkm2foQO5m6PeYo0= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= +github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -168,6 +224,11 @@ github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SA github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/speakeasy-api/jsonpath v0.6.0 h1:IhtFOV9EbXplhyRqsVhHoBmmYjblIRh5D1/g8DHMXJ8= +github.com/speakeasy-api/jsonpath v0.6.0/go.mod h1:ymb2iSkyOycmzKwbEAYPJV/yi2rSmvBCLZJcyD+VVWw= +github.com/speakeasy-api/openapi-overlay v0.10.2 h1:VOdQ03eGKeiHnpb1boZCGm7x8Haj6gST0P3SGTX95GU= +github.com/speakeasy-api/openapi-overlay v0.10.2/go.mod h1:n0iOU7AqKpNFfEt6tq7qYITC4f0yzVVdFw0S7hukemg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -176,6 +237,8 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -194,8 +257,13 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/vmware-labs/yaml-jsonpath v0.3.2 h1:/5QKeCBGdsInyDCyVNLbXyilb61MXGi9NP674f9Hobk= +github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN1BWz5V+51Ewf8k+rQ= +github.com/woodsbury/decimal128 v1.3.0 h1:8pffMNWIlC0O5vbyHWFZAt5yWvWcrHA+3ovIIjVWss0= +github.com/woodsbury/decimal128 v1.3.0/go.mod h1:C5UTmyTjW3JftjUFzOVhC20BEQa2a4ZKOB5I6Zjb+ds= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -224,27 +292,68 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= @@ -253,17 +362,33 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 5ffa29d..0b7d278 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -1,6 +1,9 @@ package apiserver +//go:generate go tool oapi-codegen -config cfg.yaml openapi.yaml + import ( + "fmt" "log" "net/http" @@ -14,7 +17,7 @@ type APIServer struct { clusterReconciler *controller.ClusterReconciler } -func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer { +func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServer, *gin.Engine) { router := gin.Default() a := &APIServer{ Server: &http.Server{ @@ -24,12 +27,7 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServe router: router, clusterReconciler: clusterReconciler, } - - a.router.POST("/clusters/:namespace/:name/createTarget", a.postCreateTarget) - a.router.GET("/clusters/:namespace/:name/plan", a.getClusterPlan) - a.router.GET("/healthz", a.getHealthz) - a.router.GET("/readyz", a.getReadyz) - return a + return a, router } func (a *APIServer) getHealthz(c *gin.Context) { @@ -40,7 +38,7 @@ func (a *APIServer) getReadyz(c *gin.Context) { c.Status(http.StatusOK) } -func (a *APIServer) getClusterPlan(c *gin.Context) { +func (a *APIServer) GetClusterPlan(c *gin.Context) { log.Printf("received GET request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) namespace, name := c.Param("namespace"), c.Param("name") plan, err := a.clusterReconciler.GetClusterPlan(namespace, name) @@ -54,7 +52,16 @@ func (a *APIServer) getClusterPlan(c *gin.Context) { // kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 // curl -X POST http://localhost:8082/clusters/gnmic-system/gnmic-controller-manager/createTarget -func (a *APIServer) postCreateTarget(c *gin.Context) { - log.Printf("received POST request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) - c.Status(202) +func (a *APIServer) CreateTargets(c *gin.Context) { + var payload []Target + if err := c.ShouldBindJSON(&payload); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + // testing + for _, target := range payload { + fmt.Printf("name: %s, address: %s, profile: %s, tags: %s\n", *target.Name, *target.Address, *target.Profile, *target.Tags) + } + + c.JSON(http.StatusOK, payload) } diff --git a/internal/apiserver/cfg.yaml b/internal/apiserver/cfg.yaml new file mode 100644 index 0000000..4bc7f02 --- /dev/null +++ b/internal/apiserver/cfg.yaml @@ -0,0 +1,6 @@ +package: apiserver +output: gen.go +generate: + gin-server: true + models: true + embedded-spec: true \ No newline at end of file diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go new file mode 100644 index 0000000..88ed5ab --- /dev/null +++ b/internal/apiserver/gen.go @@ -0,0 +1,184 @@ +// Package apiserver provides primitives to interact with the openapi HTTP API. +// +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.6.0 DO NOT EDIT. +package apiserver + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "fmt" + "net/url" + "path" + "strings" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/gin-gonic/gin" +) + +// Target defines model for Target. +type Target struct { + Address *string `json:"address,omitempty"` + Name *string `json:"name,omitempty"` + Profile *string `json:"profile,omitempty"` + Tags *[]string `json:"tags,omitempty"` +} + +// ServerInterface represents all server handlers. +type ServerInterface interface { + // Create targets in the gNMIc Operator + // (POST /createTargets) + CreateTargets(c *gin.Context) + // Get cluster plan + // (GET /plan) + GetClusterPlan(c *gin.Context) +} + +// ServerInterfaceWrapper converts contexts to parameters. +type ServerInterfaceWrapper struct { + Handler ServerInterface + HandlerMiddlewares []MiddlewareFunc + ErrorHandler func(*gin.Context, error, int) +} + +type MiddlewareFunc func(c *gin.Context) + +// CreateTargets operation middleware +func (siw *ServerInterfaceWrapper) CreateTargets(c *gin.Context) { + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.CreateTargets(c) +} + +// GetClusterPlan operation middleware +func (siw *ServerInterfaceWrapper) GetClusterPlan(c *gin.Context) { + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.GetClusterPlan(c) +} + +// GinServerOptions provides options for the Gin server. +type GinServerOptions struct { + BaseURL string + Middlewares []MiddlewareFunc + ErrorHandler func(*gin.Context, error, int) +} + +// RegisterHandlers creates http.Handler with routing matching OpenAPI spec. +func RegisterHandlers(router gin.IRouter, si ServerInterface) { + RegisterHandlersWithOptions(router, si, GinServerOptions{}) +} + +// RegisterHandlersWithOptions creates http.Handler with additional options +func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options GinServerOptions) { + errorHandler := options.ErrorHandler + if errorHandler == nil { + errorHandler = func(c *gin.Context, err error, statusCode int) { + c.JSON(statusCode, gin.H{"msg": err.Error()}) + } + } + + wrapper := ServerInterfaceWrapper{ + Handler: si, + HandlerMiddlewares: options.Middlewares, + ErrorHandler: errorHandler, + } + + router.POST(options.BaseURL+"/createTargets", wrapper.CreateTargets) + router.GET(options.BaseURL+"/plan", wrapper.GetClusterPlan) +} + +// Base64 encoded, gzipped, json marshaled Swagger object +var swaggerSpec = []string{ + + "H4sIAAAAAAAC/2ySwW7bMAyGX0XgdjRsJ7v5NgRDkMO2YMsLqDLtKLAlgaQLBIHevZDsNnWTkwmKP/nz", + "o29g/Bi8QycMzQ3YnHHUOTxp6lFSFMgHJLGY87ptCTmHcg0IDbCQdT3EApwe8elDIN/Z4fmb6D53s4Lj", + "87ZLQhPpK8R7wr9c0AjElLKu81lsJc2B/s/vg1F/A5IWT+rfr/8n9fN4gAJekdh6Bw3UZV1u0gAf0Olg", + "oYEf5basoYCg5ZzNVIZQC840ciZ4zlh87m29O7TQwG5VVgAhB+94ZratN+ljvBN0WaxDGKzJ8urCyc07", + "/BWK74QdNPCtup+pWm5ULQd65BMLaJEN2SDznosrNa/SKp6MQeZuGoaZJ0/jqOn6sYaSRWGdkjOqNcws", + "qcKgs+vlJ1nT2KPshokF6ZjKHnDU6bM2+aleEcpEDtsv5vYoysxlKo+PMca3AAAA//9/7qb/wwIAAA==", +} + +// GetSwagger returns the content of the embedded swagger specification file +// or error if failed to decode +func decodeSpec() ([]byte, error) { + zipped, err := base64.StdEncoding.DecodeString(strings.Join(swaggerSpec, "")) + if err != nil { + return nil, fmt.Errorf("error base64 decoding spec: %w", err) + } + zr, err := gzip.NewReader(bytes.NewReader(zipped)) + if err != nil { + return nil, fmt.Errorf("error decompressing spec: %w", err) + } + var buf bytes.Buffer + _, err = buf.ReadFrom(zr) + if err != nil { + return nil, fmt.Errorf("error decompressing spec: %w", err) + } + + return buf.Bytes(), nil +} + +var rawSpec = decodeSpecCached() + +// a naive cached of a decoded swagger spec +func decodeSpecCached() func() ([]byte, error) { + data, err := decodeSpec() + return func() ([]byte, error) { + return data, err + } +} + +// Constructs a synthetic filesystem for resolving external references when loading openapi specifications. +func PathToRawSpec(pathToFile string) map[string]func() ([]byte, error) { + res := make(map[string]func() ([]byte, error)) + if len(pathToFile) > 0 { + res[pathToFile] = rawSpec + } + + return res +} + +// GetSwagger returns the Swagger specification corresponding to the generated code +// in this file. The external references of Swagger specification are resolved. +// The logic of resolving external references is tightly connected to "import-mapping" feature. +// Externally referenced files must be embedded in the corresponding golang packages. +// Urls can be supported but this task was out of the scope. +func GetSwagger() (swagger *openapi3.T, err error) { + resolvePath := PathToRawSpec("") + + loader := openapi3.NewLoader() + loader.IsExternalRefsAllowed = true + loader.ReadFromURIFunc = func(loader *openapi3.Loader, url *url.URL) ([]byte, error) { + pathToFile := url.String() + pathToFile = path.Clean(pathToFile) + getSpec, ok := resolvePath[pathToFile] + if !ok { + err1 := fmt.Errorf("path not found: %s", pathToFile) + return nil, err1 + } + return getSpec() + } + var specData []byte + specData, err = rawSpec() + if err != nil { + return + } + swagger, err = loader.LoadFromData(specData) + if err != nil { + return + } + return +} diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml new file mode 100644 index 0000000..fccf33e --- /dev/null +++ b/internal/apiserver/openapi.yaml @@ -0,0 +1,45 @@ +openapi: 3.2.0 +info: + title: "gNMIc Operator REST API" + version: "0.0.1" +paths: + /plan: + get: + summary: "Get cluster plan" + operationId: "getClusterPlan" + responses: + '200': + description: "ClusterPlan returned" + /createTargets: + post: + summary: "Create targets in the gNMIc Operator" + operationId: "createTargets" + responses: + '201': + description: "Targets created successfully" + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Target' + +components: + schemas: + Target: + type: object + properties: + name: + type: string + address: + type: string # in the format "IP:port" + profile: + type: string + tags: + type: array + items: + type: string + # username: + # type: string + # password: # not sure if the password is needed + # type: string \ No newline at end of file From 5e50abaea49ea05d8f0fb7ae752e4b09bd0b0c7e Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 8 Apr 2026 20:14:12 +0000 Subject: [PATCH 005/110] add healthz and readyz interfaces to avoid pod crashing --- internal/apiserver/apiserver.go | 4 +-- internal/apiserver/gen.go | 47 ++++++++++++++++++++++++++++----- internal/apiserver/openapi.yaml | 12 +++++++++ 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 0b7d278..6d2ca5b 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -30,11 +30,11 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServ return a, router } -func (a *APIServer) getHealthz(c *gin.Context) { +func (a *APIServer) GetHealthz(c *gin.Context) { c.Status(http.StatusOK) } -func (a *APIServer) getReadyz(c *gin.Context) { +func (a *APIServer) GetReadyz(c *gin.Context) { c.Status(http.StatusOK) } diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index 88ed5ab..77b2ecb 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -29,9 +29,15 @@ type ServerInterface interface { // Create targets in the gNMIc Operator // (POST /createTargets) CreateTargets(c *gin.Context) + + // (GET /healthz) + GetHealthz(c *gin.Context) // Get cluster plan // (GET /plan) GetClusterPlan(c *gin.Context) + + // (GET /readyz) + GetReadyz(c *gin.Context) } // ServerInterfaceWrapper converts contexts to parameters. @@ -56,6 +62,19 @@ func (siw *ServerInterfaceWrapper) CreateTargets(c *gin.Context) { siw.Handler.CreateTargets(c) } +// GetHealthz operation middleware +func (siw *ServerInterfaceWrapper) GetHealthz(c *gin.Context) { + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.GetHealthz(c) +} + // GetClusterPlan operation middleware func (siw *ServerInterfaceWrapper) GetClusterPlan(c *gin.Context) { @@ -69,6 +88,19 @@ func (siw *ServerInterfaceWrapper) GetClusterPlan(c *gin.Context) { siw.Handler.GetClusterPlan(c) } +// GetReadyz operation middleware +func (siw *ServerInterfaceWrapper) GetReadyz(c *gin.Context) { + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.GetReadyz(c) +} + // GinServerOptions provides options for the Gin server. type GinServerOptions struct { BaseURL string @@ -97,18 +129,21 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options } router.POST(options.BaseURL+"/createTargets", wrapper.CreateTargets) + router.GET(options.BaseURL+"/healthz", wrapper.GetHealthz) router.GET(options.BaseURL+"/plan", wrapper.GetClusterPlan) + router.GET(options.BaseURL+"/readyz", wrapper.GetReadyz) } // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/2ySwW7bMAyGX0XgdjRsJ7v5NgRDkMO2YMsLqDLtKLAlgaQLBIHevZDsNnWTkwmKP/nz", - "o29g/Bi8QycMzQ3YnHHUOTxp6lFSFMgHJLGY87ptCTmHcg0IDbCQdT3EApwe8elDIN/Z4fmb6D53s4Lj", - "87ZLQhPpK8R7wr9c0AjElLKu81lsJc2B/s/vg1F/A5IWT+rfr/8n9fN4gAJekdh6Bw3UZV1u0gAf0Olg", - "oYEf5basoYCg5ZzNVIZQC840ciZ4zlh87m29O7TQwG5VVgAhB+94ZratN+ljvBN0WaxDGKzJ8urCyc07", - "/BWK74QdNPCtup+pWm5ULQd65BMLaJEN2SDznosrNa/SKp6MQeZuGoaZJ0/jqOn6sYaSRWGdkjOqNcws", - "qcKgs+vlJ1nT2KPshokF6ZjKHnDU6bM2+aleEcpEDtsv5vYoysxlKo+PMca3AAAA//9/7qb/wwIAAA==", + "H4sIAAAAAAAC/4xSwW7UMBD9FWvgGG3ScssNVajsAahKf2BwZhNXiW2NJ0ihyr8jTwxL6KraXGyN35t5", + "701ewIYpBk9eErQvkOxAE+r1CbknybfIIRKLI61j1zElvcoSCVpIws73sFbgcaKLD5HDyY2X3wR77eaE", + "psttSwGZcYH1XAg/nskKrLnk/Cko2UmeA/3XL0drvkVilMDm8dP3J/Px4QgV/CROLnhooTk0h5s8IETy", + "GB208OFwe2iggogyqJjaMqHQloZWYkgaS9DeLvhjBy3c7WAVMKUYfNoyu21u8mGDF/JKxhhHZ5VeP6es", + "5k/4uyjeM52ghXf1eU112VFdFvQ6n7WCjpJlF2XzWVSZzUpn0mwtpXSax3HLM83ThLz8tWGkMJw3MpDZ", + "h6mUeiAcZfiVVZb/ZB/IPcnnAnmVRpOPvcb+DF/VQh1H9G91vxvnJMQPGXbNhH/whklm9tT95/6exNgN", + "ZnS8CmHCbnnT6OOGuNJnQa/6/Q4AAP//KM8oJIIDAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index fccf33e..9471a8b 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -3,6 +3,18 @@ info: title: "gNMIc Operator REST API" version: "0.0.1" paths: + /readyz: + get: + operationId: "getReadyz" + responses: + '200': + description: "getReadyz" + /healthz: + get: + operationId: "getHealthz" + responses: + '200': + description: "getHealthz" /plan: get: summary: "Get cluster plan" From 0f230c2f848284e6a652189f07802e3010924342 Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 8 Apr 2026 20:51:58 +0000 Subject: [PATCH 006/110] move healthz and readyz back to port 8081 --- cmd/main.go | 14 ++++++- config/default/manager_config_patch.yaml | 6 +-- internal/apiserver/apiserver.go | 8 ---- internal/apiserver/gen.go | 47 +++--------------------- internal/apiserver/openapi.yaml | 12 ------ 5 files changed, 21 insertions(+), 66 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index ba4c3a3..51dba83 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -33,6 +33,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -65,10 +66,10 @@ func main() { var probeAddr string var devMode bool var apiAddr string - flag.StringVar(&apiAddr, "api-bind-address", "", "The address the operator API endpoint binds to. Disabled if empty.") + flag.StringVar(&apiAddr, "api-bind-address", ":8082", "The address the operator API endpoint binds to.") flag.BoolVar(&devMode, "dev-mode", false, "Enable development mode.") flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", "0", "Disabled: health and readiness probes are served by the gin API server.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") @@ -210,6 +211,15 @@ func main() { } //+kubebuilder:scaffold:builder + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + if apiAddr != "" { server, r := apiserver.New(apiAddr, clusterReconciler) apiserver.RegisterHandlers(r, server) diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml index d88bad0..c6d4e57 100644 --- a/config/default/manager_config_patch.yaml +++ b/config/default/manager_config_patch.yaml @@ -9,18 +9,18 @@ spec: containers: - name: manager args: - - "--health-probe-bind-address=0" + - "--health-probe-bind-address=:8081" - "--metrics-bind-address=:8080" - "--api-bind-address=:8082" - "--leader-elect" livenessProbe: httpGet: path: /healthz - port: 8082 + port: 8081 readinessProbe: httpGet: path: /readyz - port: 8082 + port: 8081 ports: - containerPort: 8080 protocol: TCP diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 6d2ca5b..3ab21d1 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -30,14 +30,6 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServ return a, router } -func (a *APIServer) GetHealthz(c *gin.Context) { - c.Status(http.StatusOK) -} - -func (a *APIServer) GetReadyz(c *gin.Context) { - c.Status(http.StatusOK) -} - func (a *APIServer) GetClusterPlan(c *gin.Context) { log.Printf("received GET request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) namespace, name := c.Param("namespace"), c.Param("name") diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index 77b2ecb..88ed5ab 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -29,15 +29,9 @@ type ServerInterface interface { // Create targets in the gNMIc Operator // (POST /createTargets) CreateTargets(c *gin.Context) - - // (GET /healthz) - GetHealthz(c *gin.Context) // Get cluster plan // (GET /plan) GetClusterPlan(c *gin.Context) - - // (GET /readyz) - GetReadyz(c *gin.Context) } // ServerInterfaceWrapper converts contexts to parameters. @@ -62,19 +56,6 @@ func (siw *ServerInterfaceWrapper) CreateTargets(c *gin.Context) { siw.Handler.CreateTargets(c) } -// GetHealthz operation middleware -func (siw *ServerInterfaceWrapper) GetHealthz(c *gin.Context) { - - for _, middleware := range siw.HandlerMiddlewares { - middleware(c) - if c.IsAborted() { - return - } - } - - siw.Handler.GetHealthz(c) -} - // GetClusterPlan operation middleware func (siw *ServerInterfaceWrapper) GetClusterPlan(c *gin.Context) { @@ -88,19 +69,6 @@ func (siw *ServerInterfaceWrapper) GetClusterPlan(c *gin.Context) { siw.Handler.GetClusterPlan(c) } -// GetReadyz operation middleware -func (siw *ServerInterfaceWrapper) GetReadyz(c *gin.Context) { - - for _, middleware := range siw.HandlerMiddlewares { - middleware(c) - if c.IsAborted() { - return - } - } - - siw.Handler.GetReadyz(c) -} - // GinServerOptions provides options for the Gin server. type GinServerOptions struct { BaseURL string @@ -129,21 +97,18 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options } router.POST(options.BaseURL+"/createTargets", wrapper.CreateTargets) - router.GET(options.BaseURL+"/healthz", wrapper.GetHealthz) router.GET(options.BaseURL+"/plan", wrapper.GetClusterPlan) - router.GET(options.BaseURL+"/readyz", wrapper.GetReadyz) } // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/4xSwW7UMBD9FWvgGG3ScssNVajsAahKf2BwZhNXiW2NJ0ihyr8jTwxL6KraXGyN35t5", - "701ewIYpBk9eErQvkOxAE+r1CbknybfIIRKLI61j1zElvcoSCVpIws73sFbgcaKLD5HDyY2X3wR77eaE", - "psttSwGZcYH1XAg/nskKrLnk/Cko2UmeA/3XL0drvkVilMDm8dP3J/Px4QgV/CROLnhooTk0h5s8IETy", - "GB208OFwe2iggogyqJjaMqHQloZWYkgaS9DeLvhjBy3c7WAVMKUYfNoyu21u8mGDF/JKxhhHZ5VeP6es", - "5k/4uyjeM52ghXf1eU112VFdFvQ6n7WCjpJlF2XzWVSZzUpn0mwtpXSax3HLM83ThLz8tWGkMJw3MpDZ", - "h6mUeiAcZfiVVZb/ZB/IPcnnAnmVRpOPvcb+DF/VQh1H9G91vxvnJMQPGXbNhH/whklm9tT95/6exNgN", - "ZnS8CmHCbnnT6OOGuNJnQa/6/Q4AAP//KM8oJIIDAAA=", + "H4sIAAAAAAAC/2ySwW7bMAyGX0XgdjRsJ7v5NgRDkMO2YMsLqDLtKLAlgaQLBIHevZDsNnWTkwmKP/nz", + "o29g/Bi8QycMzQ3YnHHUOTxp6lFSFMgHJLGY87ptCTmHcg0IDbCQdT3EApwe8elDIN/Z4fmb6D53s4Lj", + "87ZLQhPpK8R7wr9c0AjElLKu81lsJc2B/s/vg1F/A5IWT+rfr/8n9fN4gAJekdh6Bw3UZV1u0gAf0Olg", + "oYEf5basoYCg5ZzNVIZQC840ciZ4zlh87m29O7TQwG5VVgAhB+94ZratN+ljvBN0WaxDGKzJ8urCyc07", + "/BWK74QdNPCtup+pWm5ULQd65BMLaJEN2SDznosrNa/SKp6MQeZuGoaZJ0/jqOn6sYaSRWGdkjOqNcws", + "qcKgs+vlJ1nT2KPshokF6ZjKHnDU6bM2+aleEcpEDtsv5vYoysxlKo+PMca3AAAA//9/7qb/wwIAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index 9471a8b..fccf33e 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -3,18 +3,6 @@ info: title: "gNMIc Operator REST API" version: "0.0.1" paths: - /readyz: - get: - operationId: "getReadyz" - responses: - '200': - description: "getReadyz" - /healthz: - get: - operationId: "getHealthz" - responses: - '200': - description: "getHealthz" /plan: get: summary: "Get cluster plan" From 4d3dc8d949514a4a09da8707e8d805119aebcf56 Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 8 Apr 2026 21:41:21 +0000 Subject: [PATCH 007/110] url paths dynamic --- cmd/main.go | 15 +++++++++++---- config/manager/manager.yaml | 9 +++++++++ go.mod | 2 ++ go.sum | 8 ++++++++ helm/templates/deployment.yaml | 13 +++++++++++++ internal/apiserver/apiserver.go | 10 +++++++--- 6 files changed, 50 insertions(+), 7 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 51dba83..be73c0f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -221,13 +221,20 @@ func main() { } if apiAddr != "" { - server, r := apiserver.New(apiAddr, clusterReconciler) - apiserver.RegisterHandlers(r, server) + apiNamespace := os.Getenv("POD_NAMESPACE") + apiClusterName := os.Getenv("CLUSTER_NAME") + if apiNamespace == "" || apiClusterName == "" { + setupLog.Error(errors.New("missing runtime API identity"), "POD_NAMESPACE and CLUSTER_NAME must be set") + os.Exit(1) + } + apiBaseURL := "/api/v1/" + apiNamespace + "/" + apiClusterName + api, gin := apiserver.New(apiAddr, apiNamespace, apiClusterName, clusterReconciler) + apiserver.RegisterHandlersWithOptions(gin, api, apiserver.GinServerOptions{BaseURL: apiBaseURL}) err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error, 1) go func() { - err := server.Server.ListenAndServe() + err := api.Server.ListenAndServe() if err != nil && !errors.Is(err, http.ErrServerClosed) { errCh <- err } @@ -243,7 +250,7 @@ func main() { case <-ctx.Done(): shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - return server.Server.Shutdown(shutdownCtx) + return api.Server.Shutdown(shutdownCtx) } })) if err != nil { diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 2cd79f0..900f373 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -73,6 +73,15 @@ spec: - --leader-elect image: controller:latest name: manager + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/name'] securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/go.mod b/go.mod index 36c110d..d268635 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( ) require ( + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect @@ -38,6 +39,7 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/oapi-codegen/oapi-codegen/v2 v2.6.0 // indirect + github.com/oapi-codegen/runtime v1.3.1 // indirect github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect diff --git a/go.sum b/go.sum index c9059d3..81a33fb 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,12 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdB cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -144,6 +148,7 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= @@ -181,6 +186,8 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oapi-codegen/oapi-codegen/v2 v2.6.0 h1:4i+F2cvwBFZeplxCssNdLy3MhNzUD87mI3HnayHZkAU= github.com/oapi-codegen/oapi-codegen/v2 v2.6.0/go.mod h1:eWHeJSohQJIINJZzzQriVynfGsnlQVh0UkN2UYYcw4Q= +github.com/oapi-codegen/runtime v1.3.1 h1:RgDY6J4OGQLbRXhG/Xpt3vSVqYpHQS7hN4m85+5xB9g= +github.com/oapi-codegen/runtime v1.3.1/go.mod h1:kOdeacKy7t40Rclb1je37ZLFboFxh+YLy0zaPCMibPY= github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//JalHPu/3yz+De2J+4aLtSRlHiY= github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037/go.mod h1:2bpvgLBZEtENV5scfDFEtB/5+1M4hkQhDQrccEJ/qGw= github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 h1:bQx3WeLcUWy+RletIKwUIt4x3t8n2SxavmoclizMb8c= @@ -231,6 +238,7 @@ github.com/speakeasy-api/openapi-overlay v0.10.2 h1:VOdQ03eGKeiHnpb1boZCGm7x8Haj github.com/speakeasy-api/openapi-overlay v0.10.2/go.mod h1:n0iOU7AqKpNFfEt6tq7qYITC4f0yzVVdFw0S7hukemg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index b85e661..250c873 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -41,6 +41,19 @@ spec: {{- if .Values.api.port }} - --api-bind-address=:{{ .Values.api.port }} {{- end }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CLUSTER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/name'] ports: {{- if .Values.webhook.enabled }} - name: webhook diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 3ab21d1..a47fbf9 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -15,9 +15,11 @@ type APIServer struct { Server *http.Server router *gin.Engine clusterReconciler *controller.ClusterReconciler + namespace string + clusterName string } -func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServer, *gin.Engine) { +func New(addr string, namespace string, clusterName string, clusterReconciler *controller.ClusterReconciler) (*APIServer, *gin.Engine) { router := gin.Default() a := &APIServer{ Server: &http.Server{ @@ -26,14 +28,15 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServ }, router: router, clusterReconciler: clusterReconciler, + namespace: namespace, + clusterName: clusterName, } return a, router } func (a *APIServer) GetClusterPlan(c *gin.Context) { log.Printf("received GET request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) - namespace, name := c.Param("namespace"), c.Param("name") - plan, err := a.clusterReconciler.GetClusterPlan(namespace, name) + plan, err := a.clusterReconciler.GetClusterPlan(a.namespace, a.clusterName) if err != nil { c.String(404, err.Error()) return @@ -45,6 +48,7 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { // curl -X POST http://localhost:8082/clusters/gnmic-system/gnmic-controller-manager/createTarget func (a *APIServer) CreateTargets(c *gin.Context) { + log.Printf("received POST request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) var payload []Target if err := c.ShouldBindJSON(&payload); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) From 34be165c89d8be83f5f35300a1a031e5ee36b4e1 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 9 Apr 2026 11:55:32 +0000 Subject: [PATCH 008/110] improve logging for troubleshooting --- internal/apiserver/apiserver.go | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index a47fbf9..fe4f8aa 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -36,12 +36,13 @@ func New(addr string, namespace string, clusterName string, clusterReconciler *c func (a *APIServer) GetClusterPlan(c *gin.Context) { log.Printf("received GET request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) - plan, err := a.clusterReconciler.GetClusterPlan(a.namespace, a.clusterName) - if err != nil { - c.String(404, err.Error()) - return - } - c.JSON(200, plan) + // plan, err := a.clusterReconciler.GetClusterPlan(a.namespace, a.clusterName) + // if err != nil { + // c.String(404, err.Error()) + // return + // } + // c.JSON(200, plan) + c.JSON(http.StatusOK, "GetClusterPlan") } // kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 @@ -54,10 +55,24 @@ func (a *APIServer) CreateTargets(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - // testing + // For testing, to see the payload that is being sent for _, target := range payload { - fmt.Printf("name: %s, address: %s, profile: %s, tags: %s\n", *target.Name, *target.Address, *target.Profile, *target.Tags) + if target.Name != nil { + fmt.Printf("name: %s, ", *target.Name) + } + if target.Address != nil { + fmt.Printf("address: %s, ", *target.Address) + } + if target.Profile != nil { + fmt.Printf("profile: %s, ", *target.Profile) + } + if target.Tags != nil { + fmt.Printf("tags: %s", *target.Tags) + } + fmt.Printf("\n") } + // TODO: send target received from interface to autodiscover logic via channel. + c.JSON(http.StatusOK, payload) } From 7c0e2bbdde08addbb28f9d7284a7514623e091bb Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 9 Apr 2026 13:23:54 +0000 Subject: [PATCH 009/110] move gin router creation to apiserver package --- cmd/main.go | 10 +++------ internal/apiserver/apiserver.go | 40 +++++++++++++++++++-------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index be73c0f..2e3ba40 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -221,16 +221,12 @@ func main() { } if apiAddr != "" { - apiNamespace := os.Getenv("POD_NAMESPACE") - apiClusterName := os.Getenv("CLUSTER_NAME") - if apiNamespace == "" || apiClusterName == "" { - setupLog.Error(errors.New("missing runtime API identity"), "POD_NAMESPACE and CLUSTER_NAME must be set") + api, err := apiserver.New(apiAddr, clusterReconciler) + if err != nil { + setupLog.Error(err, "unable to intialize gin API server") os.Exit(1) } - apiBaseURL := "/api/v1/" + apiNamespace + "/" + apiClusterName - api, gin := apiserver.New(apiAddr, apiNamespace, apiClusterName, clusterReconciler) - apiserver.RegisterHandlersWithOptions(gin, api, apiserver.GinServerOptions{BaseURL: apiBaseURL}) err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error, 1) go func() { diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index fe4f8aa..81b8572 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -1,11 +1,13 @@ package apiserver //go:generate go tool oapi-codegen -config cfg.yaml openapi.yaml +// or use go generate ./internal/apiserver in the console (install from https://github.com/oapi-codegen/oapi-codegen) import ( + "errors" "fmt" - "log" "net/http" + "os" "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" @@ -19,7 +21,7 @@ type APIServer struct { clusterName string } -func New(addr string, namespace string, clusterName string, clusterReconciler *controller.ClusterReconciler) (*APIServer, *gin.Engine) { +func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServer, error) { router := gin.Default() a := &APIServer{ Server: &http.Server{ @@ -28,28 +30,32 @@ func New(addr string, namespace string, clusterName string, clusterReconciler *c }, router: router, clusterReconciler: clusterReconciler, - namespace: namespace, - clusterName: clusterName, + namespace: os.Getenv("POD_NAMESPACE"), + clusterName: os.Getenv("CLUSTER_NAME"), } - return a, router -} -func (a *APIServer) GetClusterPlan(c *gin.Context) { - log.Printf("received GET request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) - // plan, err := a.clusterReconciler.GetClusterPlan(a.namespace, a.clusterName) - // if err != nil { - // c.String(404, err.Error()) - // return - // } - // c.JSON(200, plan) - c.JSON(http.StatusOK, "GetClusterPlan") + if a.namespace == "" || a.clusterName == "" { + return nil, errors.New("POD_NAMESPACE and CLUSTER_NAME must be set") + } + apiBaseURL := "/api/v1/" + a.namespace + "/" + a.clusterName + RegisterHandlersWithOptions(router, a, GinServerOptions{BaseURL: apiBaseURL}) + return a, nil } // kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 -// curl -X POST http://localhost:8082/clusters/gnmic-system/gnmic-controller-manager/createTarget +// GetClusterPlan returns cluster plan +func (a *APIServer) GetClusterPlan(c *gin.Context) { + plan, err := a.clusterReconciler.GetClusterPlan(a.namespace, a.clusterName) + if err != nil { + c.String(404, err.Error()) + return + } + c.JSON(200, plan) +} + +// CreateTargets binds payload to Target struct defined in openapi.yaml and TBD... func (a *APIServer) CreateTargets(c *gin.Context) { - log.Printf("received POST request: path=%s method=%s remote=%s", c.Request.URL.Path, c.Request.Method, c.Request.RemoteAddr) var payload []Target if err := c.ShouldBindJSON(&payload); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) From 4dd7c8cd2e5a3905bb4e9ffd23536c843cc31145 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 9 Apr 2026 13:38:35 +0000 Subject: [PATCH 010/110] small refactor --- cmd/main.go | 6 +++--- config/default/manager_config_patch.yaml | 8 -------- helm/templates/deployment.yaml | 4 ---- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 2e3ba40..ec6ebe0 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -228,7 +228,7 @@ func main() { } err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - errCh := make(chan error, 1) + errCh := make(chan error) go func() { err := api.Server.ListenAndServe() if err != nil && !errors.Is(err, http.ErrServerClosed) { @@ -244,9 +244,9 @@ func main() { } return err case <-ctx.Done(): - shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - return api.Server.Shutdown(shutdownCtx) + return api.Server.Shutdown(ctx) } })) if err != nil { diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml index c6d4e57..691d2c3 100644 --- a/config/default/manager_config_patch.yaml +++ b/config/default/manager_config_patch.yaml @@ -13,14 +13,6 @@ spec: - "--metrics-bind-address=:8080" - "--api-bind-address=:8082" - "--leader-elect" - livenessProbe: - httpGet: - path: /healthz - port: 8081 - readinessProbe: - httpGet: - path: /readyz - port: 8081 ports: - containerPort: 8080 protocol: TCP diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index 250c873..314c565 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -46,10 +46,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - name: CLUSTER_NAME valueFrom: fieldRef: From e76c6f35c7d9bf06ae79a7677e09e78cc5bedebf Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 16:10:07 +0000 Subject: [PATCH 011/110] restructure discovery structs --- .../discovery/core/loader_interface.go | 2 +- .../discovery/core/message_interface.go | 8 ++++++++ internal/controller/discovery/core/types.go | 19 +++++++++++++------ 3 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 internal/controller/discovery/core/message_interface.go diff --git a/internal/controller/discovery/core/loader_interface.go b/internal/controller/discovery/core/loader_interface.go index 2b87a0a..f8e343b 100644 --- a/internal/controller/discovery/core/loader_interface.go +++ b/internal/controller/discovery/core/loader_interface.go @@ -12,7 +12,7 @@ type Loader interface { // Name returns the unique loader identifier e.g. "http_pull" Name() string - // Start begins discovery and pushes target snapshots into the out channel + // Start begins discovery and pushes target snapshots or events into the out channel // The loader must stop cleanly when ctx is cancelled Start( ctx context.Context, diff --git a/internal/controller/discovery/core/message_interface.go b/internal/controller/discovery/core/message_interface.go new file mode 100644 index 0000000..0836bc6 --- /dev/null +++ b/internal/controller/discovery/core/message_interface.go @@ -0,0 +1,8 @@ +package core + +type DiscoveryMessage interface { + isDiscoveryMessage() +} + +func (DiscoveryEvent) isDiscoveryMessage() {} +func (DiscoverySnapshot) isDiscoveryMessage() {} diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 406a22b..f56eaa2 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -9,14 +9,21 @@ type DiscoveredTarget struct { } const ( - DELETE DiscoveryEvent = 0 - CREATE DiscoveryEvent = 1 - UPDATE DiscoveryEvent = 2 + DELETE EventAction = 0 + CREATE EventAction = 1 + UPDATE EventAction = 2 ) -type DiscoveryEvent int +type EventAction int -type DiscoveryMessage struct { +type DiscoveryEvent struct { Target DiscoveredTarget - Event DiscoveryEvent + Event EventAction +} + +type DiscoverySnapshot struct { + Target []DiscoveredTarget + Event EventAction + SnapshotID string + IsLastChunk bool } From 3c18fb54fbb78db867ebba48ef3ff7e0b58e5e0a Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 16:11:42 +0000 Subject: [PATCH 012/110] offload sending logic from loader implementation --- internal/controller/discovery/core/sender.go | 69 +++++++++++++++++++ .../discovery/loaders/http_pull/loader.go | 45 ++++++------ 2 files changed, 90 insertions(+), 24 deletions(-) create mode 100644 internal/controller/discovery/core/sender.go diff --git a/internal/controller/discovery/core/sender.go b/internal/controller/discovery/core/sender.go new file mode 100644 index 0000000..84de206 --- /dev/null +++ b/internal/controller/discovery/core/sender.go @@ -0,0 +1,69 @@ +package core + +import ( + "context" +) + +// sendMessages sends discovery messages over a channel in a context-aware manner +func sendMessages(ctx context.Context, out chan<- []DiscoveryMessage, messages []DiscoveryMessage) error { + select { + case <-ctx.Done(): + return ctx.Err() + case out <- messages: + } + return nil +} + +// createDiscoverySnapshots takes a list of discovered targets and returns chunked DiscoverySnapshots +func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chunkSize int) []DiscoverySnapshot { + if chunkSize <= 0 { + chunkSize = 1 + } + + var snapshots []DiscoverySnapshot + totalTargets := len(targets) + + for i := 0; i < totalTargets; i += chunkSize { + end := i + chunkSize + if end > totalTargets { + end = totalTargets + } + + chunk := targets[i:end] + snapshots = append(snapshots, DiscoverySnapshot{ + Target: chunk, + SnapshotID: snapshotID, + IsLastChunk: (end == totalTargets), + }) + } + + return snapshots +} + +// SendSnapshot sends discovered targets as a snapshot over a channel in chunks +func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets []DiscoveredTarget, snapshotID string, chunkSize int) error { + snapshots := createDiscoverySnapshots(targets, snapshotID, chunkSize) + + for _, snapshot := range snapshots { + // Convert DiscoverySnapshot to DiscoveryMessage interface + messages := make([]DiscoveryMessage, 1) + messages[0] = snapshot + + if err := sendMessages(ctx, out, messages); err != nil { + return err + } + } + + return nil +} + +// SendEvents sends discovery messages over channel in a context-aware manner +func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent) error { + // Convert DiscoveryEvent slice to DiscoveryMessage slice + messages := make([]DiscoveryMessage, len(events)) + for i, msg := range events { + messages[i] = msg + } + + return sendMessages(ctx, out, messages) +} diff --git a/internal/controller/discovery/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go index 22868b2..94660d0 100644 --- a/internal/controller/discovery/loaders/http_pull/loader.go +++ b/internal/controller/discovery/loaders/http_pull/loader.go @@ -2,12 +2,18 @@ package http_pull import ( "context" + "fmt" "time" "sigs.k8s.io/controller-runtime/pkg/log" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/google/uuid" +) + +const ( + chunkSize = 100 ) type Loader struct{} @@ -27,7 +33,11 @@ func (l *Loader) Start( spec gnmicv1alpha1.TargetSourceSpec, out chan<- []core.DiscoveryMessage, ) error { - logger := log.FromContext(ctx).WithValues("loader", l.Name()) + logger := log.FromContext(ctx).WithValues( + "component", "loader", + "name", l.Name(), + "targetsource", targetsourceName, + ) logger.Info("HTTP pull loader started") @@ -43,35 +53,22 @@ func (l *Loader) Start( case <-ticker.C: // Example snapshot (placeholder) - targets := []core.DiscoveryMessage{ + snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceName, uuid.NewString()) + targets := []core.DiscoveredTarget{ { - Target: core.DiscoveredTarget{ - Name: "ceos1", - Address: "clab-3-nodes-ceos1:6030", - Labels: map[string]string{"TargetSource": targetsourceName}, - }, - Event: core.CREATE, + Name: "ceos1", + Address: "clab-3-nodes-ceos1:6030", + Labels: map[string]string{"TargetSource": targetsourceName}, }, { - Target: core.DiscoveredTarget{ - Name: "leaf1", - Address: "clab-3-nodes-leaf1:57400", - Labels: map[string]string{"TargetSource": targetsourceName}, - }, - Event: core.CREATE, + Name: "leaf1", + Address: "clab-3-nodes-leaf1:57400", + Labels: map[string]string{"TargetSource": targetsourceName}, }, } - // Non-blocking context-aware send - select { - case out <- targets: - logger.V(1).Info( - "emitted target snapshot", - "count", len(targets), - ) - case <-ctx.Done(): - logger.Info("context cancelled while emitting targets") - return nil + if err := core.SendSnapshot(ctx, out, targets, snapshotID, chunkSize); err != nil { + return err } } } From 86ab0f35818be90b429177c013a78b7c3fed083f Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 16:16:08 +0000 Subject: [PATCH 013/110] implement type assertion based on received message --- .../controller/discovery/target_manager.go | 75 +++++++++++++------ 1 file changed, 52 insertions(+), 23 deletions(-) diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go index 245942d..f44e33c 100644 --- a/internal/controller/discovery/target_manager.go +++ b/internal/controller/discovery/target_manager.go @@ -9,23 +9,26 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/go-logr/logr" ) -// TargetManager consumes discovered targets and applies them to Kubernetes. +// TargetManager consumes discovered targets and applies them to Kubernetes type TargetManager struct { client client.Client scheme *runtime.Scheme targetSource *gnmicv1alpha1.TargetSource in <-chan []core.DiscoveryMessage + collected map[string][]core.DiscoveredTarget } -// NewTargetManager wires a TargetManager instance. +// NewTargetManager wires a TargetManager instance func NewTargetManager(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetManager { return &TargetManager{ client: c, scheme: s, targetSource: ts, in: in, + collected: make(map[string][]core.DiscoveredTarget), } } @@ -43,28 +46,54 @@ func (m *TargetManager) Run(ctx context.Context) error { logger.Info("target manager stopped") return nil - case targets := <-m.in: - logger.Info( - "received discovered targets", - "count", len(targets), - ) + case messages := <-m.in: + for _, message := range messages { + // Type assert to determine if this is a snapshot or event + switch msg := message.(type) { + case core.DiscoverySnapshot: + // Collect snapshot chunks + logger.Info( + "received snapshot chunk", + "snapshotID", msg.SnapshotID, + "targetCount", len(msg.Target), + ) + m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Target...) + if msg.IsLastChunk { + m.processSnapshot(msg.SnapshotID, logger) + } - // List existing Target CRs owned by this TargetSource - // var existing gnmicv1alpha1.TargetList - // if err := m.client.List( - // ctx, - // &existing, - // client.MatchingLabels{ - // "gnmic.dev/targetsource": m.targetsource, - // }, - // ); err != nil { - // return err - // } - - // TODO: Target Lifecycle Management - // 1. Compare and determine which Targets to create/update/delete - // 2. Create/update/delete Target CRs accordingly - // 3. Update TargetSource status with sync results + case core.DiscoveryEvent: + // Process individual event-driven update + logger.Info( + "received discovery event", + "target", msg.Target.Name, + ) + switch msg.Event { + case core.CREATE: + logger.Info("Would create target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) + case core.UPDATE: + logger.Info("Would update target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) + case core.DELETE: + logger.Info("Would delete target", "name", msg.Target.Name) + } + } + } } } } + +// processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly +func (m *TargetManager) processSnapshot(snapshotID string, logger logr.Logger) { + targets := m.collected[snapshotID] + delete(m.collected, snapshotID) + + logger.Info("Processing full snapshot", "snapshotID", snapshotID, "totalTargets", len(targets)) + + if m.targetSource.Spec.Provider.HTTP != nil { + logger.Info("Would delete all existing targets for targetsource", "targetsource", m.targetSource.Name) + } + + for _, target := range targets { + logger.Info("Would create target", "name", target.Name, "address", target.Address, "labels", target.Labels) + } +} From efbf727aed95de42e0a582333e90262689a2a3e5 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 17:30:22 +0000 Subject: [PATCH 014/110] add http_push skeleton --- .../discovery/loaders/http_push/loader.go | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/internal/controller/discovery/loaders/http_push/loader.go b/internal/controller/discovery/loaders/http_push/loader.go index 95dc1e9..2e4ae0e 100644 --- a/internal/controller/discovery/loaders/http_push/loader.go +++ b/internal/controller/discovery/loaders/http_push/loader.go @@ -1,4 +1,53 @@ package http_push +import ( + "context" + "errors" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" + "sigs.k8s.io/controller-runtime/pkg/log" +) + // this file implements the logic receive target updates via HTTP push // REST API defined internal/apiserver + +// Loader implements the HTTP pull discovery mechanism +type Loader struct{} + +// New returns a new http_pull loader instance +func New() core.Loader { + return &Loader{} +} + +func (l *Loader) Name() string { + return "http_push" +} + +func (l *Loader) Start( + ctx context.Context, + targetsourceName string, + spec gnmicv1alpha1.TargetSourceSpec, + out chan<- []core.DiscoveryMessage, +) error { + logger := log.FromContext(ctx).WithValues( + "component", "loader", + "name", l.Name(), + "targetsource", targetsourceName, + ) + logger.Info("HTTP push loader started") + + // Input Validation of spec + if spec.Provider == nil || spec.Provider.HTTP == nil { + return errors.New("http_push loader requires spec.provider.http to be set") + } + + // Receive target updates via HTTP push + var targetEvents []core.DiscoveryEvent + + if err := core.SendEvents(ctx, out, targetEvents); err != nil { + logger.Error(err, "failed to send events") + return nil + } + return nil +} From 60a5eb3a34a741077ec465b20266ecc58eecc59b Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 17:43:15 +0000 Subject: [PATCH 015/110] refactor targetsource_controller.go --- .../controller/targetsource_controller.go | 130 +++++++++++------- lab/dev/resources/targetsources/ctest1.yaml | 3 +- 2 files changed, 83 insertions(+), 50 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 8cd6f68..9fb587f 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -55,92 +55,124 @@ type TargetSourceReconciler struct { // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx) + logger := log.FromContext(ctx).WithValues( + "Name", req.NamespacedName, + ) + targetSource, err := r.getTargetSource(ctx, req.NamespacedName) + if err != nil { + return ctrl.Result{}, err + } + + // Handle deletion with finalizer + if !targetSource.DeletionTimestamp.IsZero() { + return r.handleTargetSourceDeletion(ctx, req.NamespacedName, targetSource) + } + + // Ensure finalizer is set + if err := r.ensureFinalizer(ctx, targetSource); err != nil { + return ctrl.Result{}, err + } + + // Check if pipeline is already running + if r.isPipelineRunning(req.NamespacedName) { + return ctrl.Result{}, nil + } + + // Start discovery pipeline + if err := r.startDiscoveryPipeline(req.NamespacedName, targetSource); err != nil { + return ctrl.Result{}, err + } + + logger.Info("TargetSource pipeline started") + return ctrl.Result{}, nil +} + +// getTargetSource retrieves a TargetSource by name, handling cleanup if not found +func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key client.ObjectKey) (*gnmicv1alpha1.TargetSource, error) { var targetSource gnmicv1alpha1.TargetSource - if err := r.Get(ctx, req.NamespacedName, &targetSource); err != nil { + if err := r.Get(ctx, key, &targetSource); err != nil { // If the TargetSource no longer exists, ensure runtime cleanup if client.IgnoreNotFound(err) == nil { - r.stopDiscovery(req.NamespacedName) + r.stopDiscovery(key) } - return ctrl.Result{}, client.IgnoreNotFound(err) + return nil, client.IgnoreNotFound(err) } + return &targetSource, nil +} - logger.Info("reconciling TargetSource", "name", targetSource.Name) - - // Handle deletion with finalizer - if !targetSource.DeletionTimestamp.IsZero() { - logger.Info("TargetSource is being deleted, stopping pipeline", "name", targetSource.Name) +// handleTargetSourceDeletion stops the discovery pipeline and removes the finalizer +func (r *TargetSourceReconciler) handleTargetSourceDeletion(ctx context.Context, key client.ObjectKey, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { + logger := log.FromContext(ctx) + logger.Info("TargetSource is being deleted, stopping pipeline", "name", targetSource.Name) - r.stopDiscovery(req.NamespacedName) + r.stopDiscovery(key) - // Remove finalizer if exists - if controllerutil.ContainsFinalizer(&targetSource, targetSourceFinalizer) { - controllerutil.RemoveFinalizer(&targetSource, targetSourceFinalizer) - if err := r.Update(ctx, &targetSource); err != nil { - return ctrl.Result{}, err - } + // Remove finalizer if exists + if controllerutil.ContainsFinalizer(targetSource, targetSourceFinalizer) { + controllerutil.RemoveFinalizer(targetSource, targetSourceFinalizer) + if err := r.Update(ctx, targetSource); err != nil { + return ctrl.Result{}, err } + } - return ctrl.Result{}, nil + return ctrl.Result{}, nil +} + +// ensureFinalizer adds the finalizer if not present and updates the TargetSource +func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSource *gnmicv1alpha1.TargetSource) error { + if controllerutil.ContainsFinalizer(targetSource, targetSourceFinalizer) { + return nil } - // Ensure finalizer is set - if !controllerutil.ContainsFinalizer(&targetSource, targetSourceFinalizer) { - controllerutil.AddFinalizer(&targetSource, targetSourceFinalizer) - if err := r.Update(ctx, &targetSource); err != nil { - return ctrl.Result{}, err - } - // Requeue to continue with a clean state - return ctrl.Result{}, nil + controllerutil.AddFinalizer(targetSource, targetSourceFinalizer) + if err := r.Update(ctx, targetSource); err != nil { + return err } - // TODO: - // 1. Check if a pipeline is already running for this TargetSource - // 2. If not, create and start a new pipeline: - // a. Create a Loader based on TargetSource spec - // b. Start the Loader in a new goroutine, passing a channel for discovered targets - // c. Start a TargetManager in another goroutine to consume discovered targets and manage Target CRs - // 3. If yes, check if the spec has changed and restart the pipeline if needed + return nil +} +// isPipelineRunning checks if a discovery pipeline is already running for the given key +func (r *TargetSourceReconciler) isPipelineRunning(key client.ObjectKey) bool { r.mu.Lock() - _, exists := r.running[req.NamespacedName] - r.mu.Unlock() + defer r.mu.Unlock() - // If a targetsource loader exists, return immediately without starting - // any new loader or target manager - if exists { - return ctrl.Result{}, nil - } + _, exists := r.running[key] + return exists +} - loader, err := discovery.NewLoader(targetSource.ObjectMeta.Name, targetSource.ObjectMeta.Namespace, targetSource.Spec) +// startDiscoveryPipeline creates and starts the loader and target manager +func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, targetSource *gnmicv1alpha1.TargetSource) error { + loader, err := discovery.NewLoader( + targetSource.ObjectMeta.Name, + targetSource.ObjectMeta.Namespace, + targetSource.Spec, + ) if err != nil { - return ctrl.Result{}, err + return err } runtimeCtx, cancel := context.WithCancel(context.Background()) - targetChannel := make(chan []core.DiscoveryMessage, 10) - // start loader + // Start loader go loader.Start(runtimeCtx, targetSource.Name, targetSource.Spec, targetChannel) - // start target manager + // Start target manager manager := discovery.NewTargetManager( r.Client, r.Scheme, - &targetSource, + targetSource, targetChannel, ) go manager.Run(runtimeCtx) r.mu.Lock() - r.running[req.NamespacedName] = runningSource{cancel: cancel} + r.running[key] = runningSource{cancel: cancel} r.mu.Unlock() - logger.Info("TargetSource pipeline started", "name", targetSource.Name) - - return ctrl.Result{}, nil + return nil } // stopDiscovery stops and removes a running discovery pipeline diff --git a/lab/dev/resources/targetsources/ctest1.yaml b/lab/dev/resources/targetsources/ctest1.yaml index e0aea43..bdb1bf8 100644 --- a/lab/dev/resources/targetsources/ctest1.yaml +++ b/lab/dev/resources/targetsources/ctest1.yaml @@ -5,7 +5,8 @@ metadata: spec: provider: http: - url: http://inventory-service:8080/targets + url: http://srbsci-121:8081/api/dcim/devices/?export=test + token: nbt_PtTwUBOtEvm7.64263351281a7a34227c81e6c083c7b1ff71447348c92f5821cc2088462320f1 labels: source: inventory type: http From 1bc5d2be5e429076d9bb95578cf56eb2a42fda14 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 17:49:59 +0000 Subject: [PATCH 016/110] remove targetsource ressource to not impact main --- lab/dev/resources/targetsources/ctest1.yaml | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 lab/dev/resources/targetsources/ctest1.yaml diff --git a/lab/dev/resources/targetsources/ctest1.yaml b/lab/dev/resources/targetsources/ctest1.yaml deleted file mode 100644 index bdb1bf8..0000000 --- a/lab/dev/resources/targetsources/ctest1.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: http-discovery -spec: - provider: - http: - url: http://srbsci-121:8081/api/dcim/devices/?export=test - token: nbt_PtTwUBOtEvm7.64263351281a7a34227c81e6c083c7b1ff71447348c92f5821cc2088462320f1 - labels: - source: inventory - type: http - profile: eos \ No newline at end of file From 14e7765ae44c19dad961dc367a7a2da4ff818190 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 18:09:17 +0000 Subject: [PATCH 017/110] add batching to DiscoveryEvent's --- internal/controller/discovery/core/sender.go | 29 +++++++++++++++---- .../discovery/loaders/http_push/loader.go | 3 +- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/internal/controller/discovery/core/sender.go b/internal/controller/discovery/core/sender.go index 84de206..cc8e3c1 100644 --- a/internal/controller/discovery/core/sender.go +++ b/internal/controller/discovery/core/sender.go @@ -58,12 +58,29 @@ func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets [] } // SendEvents sends discovery messages over channel in a context-aware manner -func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent) error { - // Convert DiscoveryEvent slice to DiscoveryMessage slice - messages := make([]DiscoveryMessage, len(events)) - for i, msg := range events { - messages[i] = msg +func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent, chunkSize int) error { + if chunkSize <= 0 { + chunkSize = 1 } - return sendMessages(ctx, out, messages) + totalEvents := len(events) + for i := 0; i < totalEvents; i += chunkSize { + end := i + chunkSize + if end > totalEvents { + end = totalEvents + } + + chunk := events[i:end] + // Convert DiscoveryEvent chunk to DiscoveryMessage slice + messages := make([]DiscoveryMessage, len(chunk)) + for j, event := range chunk { + messages[j] = event + } + + if err := sendMessages(ctx, out, messages); err != nil { + return err + } + } + + return nil } diff --git a/internal/controller/discovery/loaders/http_push/loader.go b/internal/controller/discovery/loaders/http_push/loader.go index 2e4ae0e..572df1d 100644 --- a/internal/controller/discovery/loaders/http_push/loader.go +++ b/internal/controller/discovery/loaders/http_push/loader.go @@ -44,8 +44,9 @@ func (l *Loader) Start( // Receive target updates via HTTP push var targetEvents []core.DiscoveryEvent + const chunkSize = 100 - if err := core.SendEvents(ctx, out, targetEvents); err != nil { + if err := core.SendEvents(ctx, out, targetEvents, chunkSize); err != nil { logger.Error(err, "failed to send events") return nil } From b4337ead8b4eb7f3bb3b764f2141707f69698483 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 18:26:33 +0000 Subject: [PATCH 018/110] refactored sender.go --- internal/controller/discovery/core/sender.go | 65 ++++++++++--------- internal/controller/discovery/core/types.go | 2 +- .../controller/discovery/target_manager.go | 4 +- 3 files changed, 39 insertions(+), 32 deletions(-) diff --git a/internal/controller/discovery/core/sender.go b/internal/controller/discovery/core/sender.go index cc8e3c1..3e6b4aa 100644 --- a/internal/controller/discovery/core/sender.go +++ b/internal/controller/discovery/core/sender.go @@ -14,6 +14,24 @@ func sendMessages(ctx context.Context, out chan<- []DiscoveryMessage, messages [ return nil } +// forEachChunk iterates over ranges [start,end) for a total count using the provided chunkSize +func forEachChunk(total, chunkSize int, fn func(start, end int) error) error { + if chunkSize <= 0 { + chunkSize = 1 + } + + for i := 0; i < total; i += chunkSize { + end := i + chunkSize + if end > total { + end = total + } + if err := fn(i, end); err != nil { + return err + } + } + return nil +} + // createDiscoverySnapshots takes a list of discovered targets and returns chunked DiscoverySnapshots func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chunkSize int) []DiscoverySnapshot { if chunkSize <= 0 { @@ -23,19 +41,15 @@ func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chu var snapshots []DiscoverySnapshot totalTargets := len(targets) - for i := 0; i < totalTargets; i += chunkSize { - end := i + chunkSize - if end > totalTargets { - end = totalTargets - } - + _ = forEachChunk(totalTargets, chunkSize, func(i, end int) error { chunk := targets[i:end] snapshots = append(snapshots, DiscoverySnapshot{ - Target: chunk, + Targets: chunk, SnapshotID: snapshotID, IsLastChunk: (end == totalTargets), }) - } + return nil + }) return snapshots } @@ -45,7 +59,7 @@ func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets [] snapshots := createDiscoverySnapshots(targets, snapshotID, chunkSize) for _, snapshot := range snapshots { - // Convert DiscoverySnapshot to DiscoveryMessage interface + // Convert DiscoverySnapshot to DiscoveryMessage messages := make([]DiscoveryMessage, 1) messages[0] = snapshot @@ -57,30 +71,23 @@ func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets [] return nil } +func eventsToMessages(events []DiscoveryEvent) []DiscoveryMessage { + message := make([]DiscoveryMessage, len(events)) + for i, event := range events { + message[i] = event + } + return message +} + // SendEvents sends discovery messages over channel in a context-aware manner func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent, chunkSize int) error { if chunkSize <= 0 { chunkSize = 1 } + messages := eventsToMessages(events) + total := len(messages) - totalEvents := len(events) - for i := 0; i < totalEvents; i += chunkSize { - end := i + chunkSize - if end > totalEvents { - end = totalEvents - } - - chunk := events[i:end] - // Convert DiscoveryEvent chunk to DiscoveryMessage slice - messages := make([]DiscoveryMessage, len(chunk)) - for j, event := range chunk { - messages[j] = event - } - - if err := sendMessages(ctx, out, messages); err != nil { - return err - } - } - - return nil + return forEachChunk(total, chunkSize, func(i, end int) error { + return sendMessages(ctx, out, messages[i:end]) + }) } diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index f56eaa2..cac249d 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -22,7 +22,7 @@ type DiscoveryEvent struct { } type DiscoverySnapshot struct { - Target []DiscoveredTarget + Targets []DiscoveredTarget Event EventAction SnapshotID string IsLastChunk bool diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_manager.go index f44e33c..153723c 100644 --- a/internal/controller/discovery/target_manager.go +++ b/internal/controller/discovery/target_manager.go @@ -55,9 +55,9 @@ func (m *TargetManager) Run(ctx context.Context) error { logger.Info( "received snapshot chunk", "snapshotID", msg.SnapshotID, - "targetCount", len(msg.Target), + "targetCount", len(msg.Targets), ) - m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Target...) + m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Targets...) if msg.IsLastChunk { m.processSnapshot(msg.SnapshotID, logger) } From 30f3ecb6f291c55d7cdd2b73e9257189acacd106 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 17 Apr 2026 19:20:56 +0000 Subject: [PATCH 019/110] load buffer and chunk size from env variable --- cmd/main.go | 10 ++++++++-- internal/controller/discovery/core/sender.go | 11 ----------- internal/controller/discovery/core/types.go | 4 ++++ internal/controller/discovery/loader.go | 4 ++-- .../discovery/loaders/http_pull/loader.go | 16 +++++++--------- .../discovery/loaders/http_push/loader.go | 13 +++++++------ internal/controller/targetsource_controller.go | 10 +++++++++- 7 files changed, 37 insertions(+), 31 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 4c37a0d..eacdee5 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -64,6 +64,8 @@ func main() { var probeAddr string var devMode bool var apiAddr string + var discoveryChunkSize int + var discoveryBufferSize int flag.StringVar(&apiAddr, "api-bind-address", "", "The address the operator API endpoint binds to. Disabled if empty.") flag.BoolVar(&devMode, "dev-mode", false, "Enable development mode.") flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") @@ -71,6 +73,8 @@ func main() { flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") + flag.IntVar(&discoveryChunkSize, "discovery-chunk-size", 100, "Maximum number of targets/events sent in a single discovery message.") + flag.IntVar(&discoveryBufferSize, "discovery-buffer-size", 10, "Amount of discovery messages that can be queued in the channel buffer.") opts := zap.Options{ Development: devMode, } @@ -117,8 +121,10 @@ func main() { os.Exit(1) } if err := (&controller.TargetSourceReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + BufferSize: discoveryBufferSize, + ChunkSize: discoveryChunkSize, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TargetSource") os.Exit(1) diff --git a/internal/controller/discovery/core/sender.go b/internal/controller/discovery/core/sender.go index 3e6b4aa..843f30e 100644 --- a/internal/controller/discovery/core/sender.go +++ b/internal/controller/discovery/core/sender.go @@ -16,10 +16,6 @@ func sendMessages(ctx context.Context, out chan<- []DiscoveryMessage, messages [ // forEachChunk iterates over ranges [start,end) for a total count using the provided chunkSize func forEachChunk(total, chunkSize int, fn func(start, end int) error) error { - if chunkSize <= 0 { - chunkSize = 1 - } - for i := 0; i < total; i += chunkSize { end := i + chunkSize if end > total { @@ -34,10 +30,6 @@ func forEachChunk(total, chunkSize int, fn func(start, end int) error) error { // createDiscoverySnapshots takes a list of discovered targets and returns chunked DiscoverySnapshots func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chunkSize int) []DiscoverySnapshot { - if chunkSize <= 0 { - chunkSize = 1 - } - var snapshots []DiscoverySnapshot totalTargets := len(targets) @@ -81,9 +73,6 @@ func eventsToMessages(events []DiscoveryEvent) []DiscoveryMessage { // SendEvents sends discovery messages over channel in a context-aware manner func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent, chunkSize int) error { - if chunkSize <= 0 { - chunkSize = 1 - } messages := eventsToMessages(events) total := len(messages) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index cac249d..69a407e 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -1,5 +1,9 @@ package core +type LoaderConfig struct { + ChunkSize int +} + // DiscoveredTarget represents a target discovered from an external source // before it is materialized as a Kubernetes Target CR type DiscoveredTarget struct { diff --git a/internal/controller/discovery/loader.go b/internal/controller/discovery/loader.go index ad1e83f..e0834c0 100644 --- a/internal/controller/discovery/loader.go +++ b/internal/controller/discovery/loader.go @@ -9,12 +9,12 @@ import ( ) // NewLoader creates a loader by name -func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) { +func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpec, cfg core.LoaderConfig) (core.Loader, error) { loaderName := namespace + "/" + name switch { case spec.Provider.HTTP != nil: - return http_pull.New(), nil + return http_pull.New(cfg), nil case spec.Provider.Consul != nil: return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName) default: diff --git a/internal/controller/discovery/loaders/http_pull/loader.go b/internal/controller/discovery/loaders/http_pull/loader.go index 94660d0..8213c8a 100644 --- a/internal/controller/discovery/loaders/http_pull/loader.go +++ b/internal/controller/discovery/loaders/http_pull/loader.go @@ -12,15 +12,13 @@ import ( "github.com/google/uuid" ) -const ( - chunkSize = 100 -) - -type Loader struct{} +type Loader struct { + cfg core.LoaderConfig +} -// New instantiates the http_pull loader -func New() core.Loader { - return &Loader{} +// New instantiates the http_pull loader with the provided config +func New(cfg core.LoaderConfig) core.Loader { + return &Loader{cfg: cfg} } func (l *Loader) Name() string { @@ -67,7 +65,7 @@ func (l *Loader) Start( }, } - if err := core.SendSnapshot(ctx, out, targets, snapshotID, chunkSize); err != nil { + if err := core.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil { return err } } diff --git a/internal/controller/discovery/loaders/http_push/loader.go b/internal/controller/discovery/loaders/http_push/loader.go index 572df1d..025176f 100644 --- a/internal/controller/discovery/loaders/http_push/loader.go +++ b/internal/controller/discovery/loaders/http_push/loader.go @@ -13,11 +13,13 @@ import ( // REST API defined internal/apiserver // Loader implements the HTTP pull discovery mechanism -type Loader struct{} +type Loader struct{ + cfg core.LoaderConfig +} -// New returns a new http_pull loader instance -func New() core.Loader { - return &Loader{} +// New returns a new http_push loader instance configured with cfg +func New(cfg core.LoaderConfig) core.Loader { + return &Loader{cfg: cfg} } func (l *Loader) Name() string { @@ -44,9 +46,8 @@ func (l *Loader) Start( // Receive target updates via HTTP push var targetEvents []core.DiscoveryEvent - const chunkSize = 100 - if err := core.SendEvents(ctx, out, targetEvents, chunkSize); err != nil { + if err := core.SendEvents(ctx, out, targetEvents, l.cfg.ChunkSize); err != nil { logger.Error(err, "failed to send events") return nil } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 9fb587f..fce6742 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -45,6 +45,9 @@ type TargetSourceReconciler struct { mu sync.Mutex running map[client.ObjectKey]runningSource + + BufferSize int + ChunkSize int } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete @@ -144,17 +147,22 @@ func (r *TargetSourceReconciler) isPipelineRunning(key client.ObjectKey) bool { // startDiscoveryPipeline creates and starts the loader and target manager func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, targetSource *gnmicv1alpha1.TargetSource) error { + cfg := core.LoaderConfig{ + ChunkSize: r.ChunkSize, + } + loader, err := discovery.NewLoader( targetSource.ObjectMeta.Name, targetSource.ObjectMeta.Namespace, targetSource.Spec, + cfg, ) if err != nil { return err } runtimeCtx, cancel := context.WithCancel(context.Background()) - targetChannel := make(chan []core.DiscoveryMessage, 10) + targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) // Start loader go loader.Start(runtimeCtx, targetSource.Name, targetSource.Spec, targetChannel) From dbbcb4f1b47f8f90f86628dae0ded11dfc4c8816 Mon Sep 17 00:00:00 2001 From: Janooski Date: Mon, 20 Apr 2026 14:53:12 +0000 Subject: [PATCH 020/110] can start push loader --- Makefile | 2 +- api/v1alpha1/targetsource_types.go | 7 +++- .../operator.gnmic.dev_targetsources.yaml | 5 +++ internal/controller/discovery/loader.go | 3 ++ .../controller/discovery/loaders/all/all.go | 3 +- .../discovery/loaders/http_push/loader.go | 32 +++++++++++++++++-- .../loaders/http_push/loader_test.go | 2 +- .../resources/targetsources/ctestPull.yaml | 12 +++++++ lab/dev/temp | 0 9 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 lab/dev/resources/targetsources/ctestPull.yaml delete mode 100644 lab/dev/temp diff --git a/Makefile b/Makefile index fdcc2b2..46f1e49 100644 --- a/Makefile +++ b/Makefile @@ -282,7 +282,7 @@ delete-outputs-dev-lab: ## Delete the outputs for the development lab cluster kubectl delete -f lab/dev/resources/outputs .PHONY: apply-pipelines-dev-lab -apply-pipelines-dev-lab: ## Apply the pipelines for the development lab cluster + §apply-pipelines-dev-lab: ## Apply the pipelines for the development lab cluster kubectl apply -f lab/dev/resources/pipelines .PHONY: delete-pipelines-dev-lab diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index 3cf029b..057425c 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -35,11 +35,16 @@ type TargetSourceSpec struct { // +kubebuilder:validation:MaxProperties=1 type ProviderSpec struct { HTTP *HTTPConfig `json:"http,omitempty"` + PULL *PULLConfig `json:"pull,omitempty"` Consul *ConsulConfig `json:"consul,omitempty"` } +type PULLConfig struct { + URL string `json:"url,omitempty"` // Placeholder for future settings, URL is not actually needed for this interface +} + type HTTPConfig struct { - URL string `json:"url,omitempty"` + URL string `json:"url,omitempty"` } type ConsulConfig struct { diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 0129a88..3369de7 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -59,6 +59,11 @@ spec: url: type: string type: object + pull: + properties: + url: + type: string + type: object type: object type: type: string diff --git a/internal/controller/discovery/loader.go b/internal/controller/discovery/loader.go index ad1e83f..24236ac 100644 --- a/internal/controller/discovery/loader.go +++ b/internal/controller/discovery/loader.go @@ -6,6 +6,7 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/loaders/http_pull" + "github.com/gnmic/operator/internal/controller/discovery/loaders/http_push" ) // NewLoader creates a loader by name @@ -15,6 +16,8 @@ func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpe switch { case spec.Provider.HTTP != nil: return http_pull.New(), nil + case spec.Provider.PULL != nil: + return http_push.New(), nil case spec.Provider.Consul != nil: return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName) default: diff --git a/internal/controller/discovery/loaders/all/all.go b/internal/controller/discovery/loaders/all/all.go index c53b98a..731bffa 100644 --- a/internal/controller/discovery/loaders/all/all.go +++ b/internal/controller/discovery/loaders/all/all.go @@ -2,5 +2,6 @@ package all import ( _ "github.com/gnmic/operator/internal/controller/discovery/loaders/http_pull" - // _ "github.com/gnmic/operator/internal/controller/targetsource/loaders/http_push" + _ "github.com/gnmic/operator/internal/controller/discovery/loaders/http_push" ) + diff --git a/internal/controller/discovery/loaders/http_push/loader.go b/internal/controller/discovery/loaders/http_push/loader.go index 95dc1e9..df7961f 100644 --- a/internal/controller/discovery/loaders/http_push/loader.go +++ b/internal/controller/discovery/loaders/http_push/loader.go @@ -1,4 +1,32 @@ package http_push -// this file implements the logic receive target updates via HTTP push -// REST API defined internal/apiserver +import ( + "context" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +type Loader struct{} + +// New instantiates the http_pull loader +func New() core.Loader { + return &Loader{} +} + +func (l *Loader) Name() string { + return "push" +} + +func (l *Loader) Start( + ctx context.Context, + targetsourceName string, + spec gnmicv1alpha1.TargetSourceSpec, + out chan<- []core.DiscoveryMessage, +) error { + logger := log.FromContext(ctx).WithValues("loader", l.Name()) + logger.Info("Push loader started") + + return nil +} diff --git a/internal/controller/discovery/loaders/http_push/loader_test.go b/internal/controller/discovery/loaders/http_push/loader_test.go index bb7d848..c75a5a0 100644 --- a/internal/controller/discovery/loaders/http_push/loader_test.go +++ b/internal/controller/discovery/loaders/http_push/loader_test.go @@ -1 +1 @@ -package http_push +package http_push \ No newline at end of file diff --git a/lab/dev/resources/targetsources/ctestPull.yaml b/lab/dev/resources/targetsources/ctestPull.yaml new file mode 100644 index 0000000..118e729 --- /dev/null +++ b/lab/dev/resources/targetsources/ctestPull.yaml @@ -0,0 +1,12 @@ +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: pull-loader +spec: + provider: + pull: + url: http://inventory-service:8080/targets + labels: + source: inventory + type: pull + profile: eos \ No newline at end of file diff --git a/lab/dev/temp b/lab/dev/temp deleted file mode 100644 index e69de29..0000000 From 3aae153c93e11e4080da8fbf1495704dd8cc1f78 Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 22 Apr 2026 09:30:40 +0000 Subject: [PATCH 021/110] pass discovery message to loader --- go.mod | 4 +-- go.sum | 14 ++++----- internal/apiserver/apiserver.go | 31 +++++++------------ .../discovery/loaders/http_push/loader.go | 21 +++++++++++++ 4 files changed, 40 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index d268635..507554e 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.25.5 require ( github.com/cert-manager/cert-manager v1.19.3 + github.com/getkin/kin-openapi v0.133.0 github.com/go-logr/logr v1.4.3 github.com/onsi/ginkgo/v2 v2.28.1 github.com/onsi/gomega v1.39.0 @@ -18,14 +19,12 @@ require ( ) require ( - github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect github.com/gabriel-vasile/mimetype v1.4.12 // indirect - github.com/getkin/kin-openapi v0.133.0 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -39,7 +38,6 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/oapi-codegen/oapi-codegen/v2 v2.6.0 // indirect - github.com/oapi-codegen/runtime v1.3.1 // indirect github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect diff --git a/go.sum b/go.sum index 81a33fb..63bb743 100644 --- a/go.sum +++ b/go.sum @@ -2,12 +2,8 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdB cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= -github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= -github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -107,6 +103,8 @@ github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= @@ -148,7 +146,6 @@ github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= @@ -183,11 +180,10 @@ github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwd github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oapi-codegen/oapi-codegen/v2 v2.6.0 h1:4i+F2cvwBFZeplxCssNdLy3MhNzUD87mI3HnayHZkAU= github.com/oapi-codegen/oapi-codegen/v2 v2.6.0/go.mod h1:eWHeJSohQJIINJZzzQriVynfGsnlQVh0UkN2UYYcw4Q= -github.com/oapi-codegen/runtime v1.3.1 h1:RgDY6J4OGQLbRXhG/Xpt3vSVqYpHQS7hN4m85+5xB9g= -github.com/oapi-codegen/runtime v1.3.1/go.mod h1:kOdeacKy7t40Rclb1je37ZLFboFxh+YLy0zaPCMibPY= github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//JalHPu/3yz+De2J+4aLtSRlHiY= github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037/go.mod h1:2bpvgLBZEtENV5scfDFEtB/5+1M4hkQhDQrccEJ/qGw= github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 h1:bQx3WeLcUWy+RletIKwUIt4x3t8n2SxavmoclizMb8c= @@ -195,6 +191,7 @@ github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90/go.mod h1:y5+oSEHCPT github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= @@ -231,6 +228,7 @@ github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SA github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/speakeasy-api/jsonpath v0.6.0 h1:IhtFOV9EbXplhyRqsVhHoBmmYjblIRh5D1/g8DHMXJ8= github.com/speakeasy-api/jsonpath v0.6.0/go.mod h1:ymb2iSkyOycmzKwbEAYPJV/yi2rSmvBCLZJcyD+VVWw= @@ -238,7 +236,6 @@ github.com/speakeasy-api/openapi-overlay v0.10.2 h1:VOdQ03eGKeiHnpb1boZCGm7x8Haj github.com/speakeasy-api/openapi-overlay v0.10.2/go.mod h1:n0iOU7AqKpNFfEt6tq7qYITC4f0yzVVdFw0S7hukemg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -389,6 +386,7 @@ gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWM gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 81b8572..e7314ac 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -5,12 +5,13 @@ package apiserver import ( "errors" - "fmt" "net/http" "os" "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" + "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/gnmic/operator/internal/controller/discovery/loaders/http_push" ) type APIServer struct { @@ -61,24 +62,16 @@ func (a *APIServer) CreateTargets(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - // For testing, to see the payload that is being sent - for _, target := range payload { - if target.Name != nil { - fmt.Printf("name: %s, ", *target.Name) - } - if target.Address != nil { - fmt.Printf("address: %s, ", *target.Address) - } - if target.Profile != nil { - fmt.Printf("profile: %s, ", *target.Profile) - } - if target.Tags != nil { - fmt.Printf("tags: %s", *target.Tags) - } - fmt.Printf("\n") + targets := []core.DiscoveryMessage{ + { + Target: core.DiscoveredTarget{ + Name: *payload[0].Name, + Address: *payload[0].Address + ":6030", + Labels: map[string]string{"TargetSource": "targetsourceName"}, + }, + Event: core.CREATE, + }, } - - // TODO: send target received from interface to autodiscover logic via channel. - + http_push.SendTargetToLoader(targets) c.JSON(http.StatusOK, payload) } diff --git a/internal/controller/discovery/loaders/http_push/loader.go b/internal/controller/discovery/loaders/http_push/loader.go index df7961f..25aa9f8 100644 --- a/internal/controller/discovery/loaders/http_push/loader.go +++ b/internal/controller/discovery/loaders/http_push/loader.go @@ -2,7 +2,9 @@ package http_push import ( "context" + "fmt" + "github.com/bytedance/gopkg/util/logger" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" "sigs.k8s.io/controller-runtime/pkg/log" @@ -30,3 +32,22 @@ func (l *Loader) Start( return nil } + +func SendTargetToLoader(dm []core.DiscoveryMessage) { + logger.Info("SendTargetToLoader %s", dm) + // for _, target := range payload { + // if target.Name != nil { + // fmt.Printf("name: %s, ", *target.Name) + // } + // if target.Address != nil { + // fmt.Printf("address: %s, ", *target.Address) + // } + // if target.Profile != nil { + // fmt.Printf("profile: %s, ", *target.Profile) + // } + // if target.Tags != nil { + // fmt.Printf("tags: %s", *target.Tags) + // } + fmt.Printf("SentTargetToLoader called") + //} +} From 586001e963125cde484ddead4e16ef11c4939c7b Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 22 Apr 2026 12:58:44 +0000 Subject: [PATCH 022/110] rename file to helpers --- internal/controller/discovery/core/{sender.go => helpers.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename internal/controller/discovery/core/{sender.go => helpers.go} (100%) diff --git a/internal/controller/discovery/core/sender.go b/internal/controller/discovery/core/helpers.go similarity index 100% rename from internal/controller/discovery/core/sender.go rename to internal/controller/discovery/core/helpers.go From 7430815bb78b417702c6df5b8e85377e63193a4b Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 22 Apr 2026 13:03:03 +0000 Subject: [PATCH 023/110] rebuild and reformat --- api/v1alpha1/zz_generated.deepcopy.go | 2 +- internal/controller/discovery/loaders/push/loader.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 3848412..61e81fd 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1alpha1 import ( - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/internal/controller/discovery/loaders/push/loader.go b/internal/controller/discovery/loaders/push/loader.go index 5b00081..ec70830 100644 --- a/internal/controller/discovery/loaders/push/loader.go +++ b/internal/controller/discovery/loaders/push/loader.go @@ -13,7 +13,7 @@ import ( // REST API defined internal/apiserver // Loader implements the HTTP pull discovery mechanism -type Loader struct{ +type Loader struct { cfg core.LoaderConfig } From 255a1f3facb9f3c6b4e4ae17b4ad1afae0bcd0bd Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 23 Apr 2026 07:15:38 +0000 Subject: [PATCH 024/110] consolidate pull and push to http --- internal/controller/discovery/loader.go | 4 +- .../controller/discovery/loaders/all/all.go | 3 +- .../loaders/{pull => http}/loader.go | 10 ++-- .../discovery/loaders/http/loader_test.go | 1 + .../discovery/loaders/pull/loader_test.go | 1 - .../discovery/loaders/push/loader.go | 55 ------------------- .../discovery/loaders/push/loader_test.go | 1 - 7 files changed, 9 insertions(+), 66 deletions(-) rename internal/controller/discovery/loaders/{pull => http}/loader.go (89%) create mode 100644 internal/controller/discovery/loaders/http/loader_test.go delete mode 100644 internal/controller/discovery/loaders/pull/loader_test.go delete mode 100644 internal/controller/discovery/loaders/push/loader.go delete mode 100644 internal/controller/discovery/loaders/push/loader_test.go diff --git a/internal/controller/discovery/loader.go b/internal/controller/discovery/loader.go index 64dc8d3..42ce8da 100644 --- a/internal/controller/discovery/loader.go +++ b/internal/controller/discovery/loader.go @@ -5,7 +5,7 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" - pull "github.com/gnmic/operator/internal/controller/discovery/loaders/pull" + http "github.com/gnmic/operator/internal/controller/discovery/loaders/http" ) // NewLoader creates a loader by name @@ -14,7 +14,7 @@ func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpe switch { case spec.Provider.HTTP != nil: - return pull.New(cfg), nil + return http.New(cfg), nil case spec.Provider.Consul != nil: return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName) default: diff --git a/internal/controller/discovery/loaders/all/all.go b/internal/controller/discovery/loaders/all/all.go index d05604b..3590cda 100644 --- a/internal/controller/discovery/loaders/all/all.go +++ b/internal/controller/discovery/loaders/all/all.go @@ -1,6 +1,5 @@ package all import ( - _ "github.com/gnmic/operator/internal/controller/discovery/loaders/pull" - // _ "github.com/gnmic/operator/internal/controller/targetsource/loaders/push" + _ "github.com/gnmic/operator/internal/controller/discovery/loaders/http" ) diff --git a/internal/controller/discovery/loaders/pull/loader.go b/internal/controller/discovery/loaders/http/loader.go similarity index 89% rename from internal/controller/discovery/loaders/pull/loader.go rename to internal/controller/discovery/loaders/http/loader.go index 729233d..f014a2f 100644 --- a/internal/controller/discovery/loaders/pull/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -1,4 +1,4 @@ -package pull +package http import ( "context" @@ -16,13 +16,13 @@ type Loader struct { cfg core.LoaderConfig } -// New instantiates the pull loader with the provided config +// New instantiates the http loader with the provided config func New(cfg core.LoaderConfig) core.Loader { return &Loader{cfg: cfg} } func (l *Loader) Name() string { - return "pull" + return "http" } func (l *Loader) Start( @@ -37,7 +37,7 @@ func (l *Loader) Start( "targetsource", targetsourceName, ) - logger.Info("HTTP pull loader started") + logger.Info("HTTP loader started") // Only for debugging: emit a static snapshot every 30 seconds ticker := time.NewTicker(30 * time.Second) @@ -46,7 +46,7 @@ func (l *Loader) Start( for { select { case <-ctx.Done(): - logger.Info("HTTP pull loader stopped") + logger.Info("HTTP loader stopped") return nil case <-ticker.C: diff --git a/internal/controller/discovery/loaders/http/loader_test.go b/internal/controller/discovery/loaders/http/loader_test.go new file mode 100644 index 0000000..d02cfda --- /dev/null +++ b/internal/controller/discovery/loaders/http/loader_test.go @@ -0,0 +1 @@ +package http diff --git a/internal/controller/discovery/loaders/pull/loader_test.go b/internal/controller/discovery/loaders/pull/loader_test.go deleted file mode 100644 index 0493bec..0000000 --- a/internal/controller/discovery/loaders/pull/loader_test.go +++ /dev/null @@ -1 +0,0 @@ -package pull diff --git a/internal/controller/discovery/loaders/push/loader.go b/internal/controller/discovery/loaders/push/loader.go deleted file mode 100644 index ec70830..0000000 --- a/internal/controller/discovery/loaders/push/loader.go +++ /dev/null @@ -1,55 +0,0 @@ -package push - -import ( - "context" - "errors" - - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" - "github.com/gnmic/operator/internal/controller/discovery/core" - "sigs.k8s.io/controller-runtime/pkg/log" -) - -// this file implements the logic receive target updates via HTTP push -// REST API defined internal/apiserver - -// Loader implements the HTTP pull discovery mechanism -type Loader struct { - cfg core.LoaderConfig -} - -// New returns a new http_push loader instance configured with cfg -func New(cfg core.LoaderConfig) core.Loader { - return &Loader{cfg: cfg} -} - -func (l *Loader) Name() string { - return "http_push" -} - -func (l *Loader) Start( - ctx context.Context, - targetsourceName string, - spec gnmicv1alpha1.TargetSourceSpec, - out chan<- []core.DiscoveryMessage, -) error { - logger := log.FromContext(ctx).WithValues( - "component", "loader", - "name", l.Name(), - "targetsource", targetsourceName, - ) - logger.Info("HTTP push loader started") - - // Input Validation of spec - if spec.Provider == nil || spec.Provider.HTTP == nil { - return errors.New("http_push loader requires spec.provider.http to be set") - } - - // Receive target updates via HTTP push - var targetEvents []core.DiscoveryEvent - - if err := core.SendEvents(ctx, out, targetEvents, l.cfg.ChunkSize); err != nil { - logger.Error(err, "failed to send events") - return nil - } - return nil -} diff --git a/internal/controller/discovery/loaders/push/loader_test.go b/internal/controller/discovery/loaders/push/loader_test.go deleted file mode 100644 index 63fdf61..0000000 --- a/internal/controller/discovery/loaders/push/loader_test.go +++ /dev/null @@ -1 +0,0 @@ -package push From 9316ca3edffaab10f02ea90ad8e5c4bc2c524c0f Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 23 Apr 2026 09:16:22 +0000 Subject: [PATCH 025/110] create multiple core.DiscoveredTarget --- internal/apiserver/apiserver.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index e7314ac..d760d3b 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -62,15 +62,16 @@ func (a *APIServer) CreateTargets(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - targets := []core.DiscoveryMessage{ - { + targets := make([]core.DiscoveryMessage, 0, len(payload)) + for _, target := range payload { + targets = append(targets, core.DiscoveryMessage{ Target: core.DiscoveredTarget{ - Name: *payload[0].Name, - Address: *payload[0].Address + ":6030", - Labels: map[string]string{"TargetSource": "targetsourceName"}, + Name: *target.Name, + Address: *target.Address, + Labels: map[string]string{"TargetSource": "*target.Tags to be"}, }, Event: core.CREATE, - }, + }) } http_push.SendTargetToLoader(targets) c.JSON(http.StatusOK, payload) From bd2b45f63366eaaba0170c37e1783e018049eaca Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 23 Apr 2026 12:34:50 +0000 Subject: [PATCH 026/110] rename target manager to target applier --- .../controller/discovery/{target_manager.go => target_applier.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename internal/controller/discovery/{target_manager.go => target_applier.go} (100%) diff --git a/internal/controller/discovery/target_manager.go b/internal/controller/discovery/target_applier.go similarity index 100% rename from internal/controller/discovery/target_manager.go rename to internal/controller/discovery/target_applier.go From 5a561a768f1a2d17e1ed09a40b82884bc512527f Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 23 Apr 2026 13:07:25 +0000 Subject: [PATCH 027/110] implement a generic registry --- .../controller/discovery/registry/registry.go | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 internal/controller/discovery/registry/registry.go diff --git a/internal/controller/discovery/registry/registry.go b/internal/controller/discovery/registry/registry.go new file mode 100644 index 0000000..7da0757 --- /dev/null +++ b/internal/controller/discovery/registry/registry.go @@ -0,0 +1,61 @@ +package registry + +import ( + "fmt" + "sync" +) + +/* USAGE + +// create registry once in main.go +discoveryReg := discovery.NewRegistry[[]core.DiscoveryMessage]() + +// inside targetsource controller, when starting discovery pipeline: +key := fmt.Sprintf("%s/%s", spec.Namespace, targetsourceName) +if err := discoveryReg.Register(key, out); err != nil { + logger.Error(err, "could not register loader") + return err +} +defer discoveryReg.Unregister(key) + +// CHECK REGISTRY +ch, ok := discoveryReg.Get(ns + "/" + ts) +if !ok { + http.Error(w, "no loader for targetsource", http.StatusNotFound) + return +} +// then deliver payload to ch +*/ + +// Registry is a thread-safe map: key -> channel of T. +type Registry[T any] struct { + mu sync.RWMutex + m map[string]chan<- T +} + +func NewRegistry[T any]() *Registry[T] { + return &Registry[T]{m: make(map[string]chan<- T)} +} + +func (r *Registry[T]) Register(key string, ch chan<- T) error { + r.mu.Lock() + defer r.mu.Unlock() + if _, exists := r.m[key]; exists { + return fmt.Errorf("already registered: %s", key) + } + r.m[key] = ch + return nil +} + +func (r *Registry[T]) Unregister(key string) { + r.mu.Lock() + delete(r.m, key) + r.mu.Unlock() +} + +func (r *Registry[T]) Get(key string) (chan<- T, bool) { + r.mu.RLock() + ch, ok := r.m[key] + r.mu.RUnlock() + return ch, ok +} From f5481b8f9c7627d9c499c9156afdb3c7c2346146 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 23 Apr 2026 13:08:01 +0000 Subject: [PATCH 028/110] add a discoveryTegistry to share targetchannel between apiserver and target manager --- cmd/main.go | 14 +++++++++---- internal/apiserver/apiserver.go | 4 ++++ .../controller/targetsource_controller.go | 20 ++++++++++++++++--- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index eacdee5..5cf8169 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -40,6 +40,8 @@ import ( operatorv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/apiserver" "github.com/gnmic/operator/internal/controller" + "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/gnmic/operator/internal/controller/discovery/registry" webhookv1alpha1 "github.com/gnmic/operator/internal/webhook/v1alpha1" //+kubebuilder:scaffold:imports ) @@ -83,6 +85,8 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + discoveryRegistry := registry.NewRegistry[[]core.DiscoveryMessage]() + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{BindAddress: metricsAddr}, @@ -121,10 +125,11 @@ func main() { os.Exit(1) } if err := (&controller.TargetSourceReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - BufferSize: discoveryBufferSize, - ChunkSize: discoveryChunkSize, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + BufferSize: discoveryBufferSize, + ChunkSize: discoveryChunkSize, + DiscoveryRegistry: discoveryRegistry, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TargetSource") os.Exit(1) @@ -226,6 +231,7 @@ func main() { if apiAddr != "" { apiServer := apiserver.New(apiAddr, clusterReconciler) + apiServer.DiscoveryRegistry = discoveryRegistry err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error) go func() { diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index f31abaa..b84eb9a 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -5,11 +5,15 @@ import ( "net/http" "github.com/gnmic/operator/internal/controller" + "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/gnmic/operator/internal/controller/discovery/registry" ) type APIServer struct { Server *http.Server clusterReconciler *controller.ClusterReconciler + + DiscoveryRegistry *registry.Registry[[]core.DiscoveryMessage] } func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer { diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index fce6742..c714acc 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -30,6 +30,7 @@ import ( "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" _ "github.com/gnmic/operator/internal/controller/discovery/loaders/all" + "github.com/gnmic/operator/internal/controller/discovery/registry" ) const targetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer" @@ -48,6 +49,8 @@ type TargetSourceReconciler struct { BufferSize int ChunkSize int + + DiscoveryRegistry *registry.Registry[[]core.DiscoveryMessage] } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete @@ -164,6 +167,12 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, ta runtimeCtx, cancel := context.WithCancel(context.Background()) targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) + registryKey := key.Namespace + "/" + key.Name + if err := r.DiscoveryRegistry.Register(registryKey, targetChannel); err != nil { + cancel() + return err + } + // Start loader go loader.Start(runtimeCtx, targetSource.Name, targetSource.Spec, targetChannel) @@ -187,12 +196,17 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, ta // for the given TargetSource key func (r *TargetSourceReconciler) stopDiscovery(key client.ObjectKey) { r.mu.Lock() - defer r.mu.Unlock() - - if running, ok := r.running[key]; ok { + running, ok := r.running[key] + if ok { running.cancel() delete(r.running, key) } + r.mu.Unlock() + + if ok { + registryKey := key.Namespace + "/" + key.Name + r.DiscoveryRegistry.Unregister(registryKey) + } } // SetupWithManager sets up the controller with the Manager. From e73a2533f915e1747f89ef6bfc28563365466d4f Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 23 Apr 2026 13:45:20 +0000 Subject: [PATCH 029/110] pull interface only in apiserver --- internal/apiserver/apiserver.go | 20 +++-- .../discovery/loaders/push/loader.go | 78 ------------------- .../discovery/loaders/push/loader_test.go | 1 - 3 files changed, 9 insertions(+), 90 deletions(-) delete mode 100644 internal/controller/discovery/loaders/push/loader.go delete mode 100644 internal/controller/discovery/loaders/push/loader_test.go diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index b984aa5..e4aaf97 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -11,7 +11,6 @@ import ( "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery/core" - "github.com/gnmic/operator/internal/controller/discovery/loaders/http_push" ) type APIServer struct { @@ -55,24 +54,23 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { c.JSON(200, plan) } -// CreateTargets binds payload to Target struct defined in openapi.yaml and TBD... +// CreateTargets binds payload to Target struct defined in openapi.yaml and sends it to pull loader func (a *APIServer) CreateTargets(c *gin.Context) { var payload []Target if err := c.ShouldBindJSON(&payload); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - targets := make([]core.DiscoveryMessage, 0, len(payload)) + targets := []core.DiscoveredTarget{} for _, target := range payload { - targets = append(targets, core.DiscoveryMessage{ - Target: core.DiscoveredTarget{ - Name: *target.Name, - Address: *target.Address, - Labels: map[string]string{"TargetSource": "*target.Tags to be done"}, - }, - Event: core.CREATE, + targets = append(targets, core.DiscoveredTarget{ + Name: *target.Name, + Address: *target.Address, + Labels: map[string]string{"key": "Is this a tag?"}, }) } - http_push.SendTargetToLoader(targets) + + // discovery / core / helpers / sendEvents to send received udpates to TagetManager + // loader push not needed c.JSON(http.StatusOK, payload) } diff --git a/internal/controller/discovery/loaders/push/loader.go b/internal/controller/discovery/loaders/push/loader.go deleted file mode 100644 index 87938a6..0000000 --- a/internal/controller/discovery/loaders/push/loader.go +++ /dev/null @@ -1,78 +0,0 @@ -package push - -import ( - "context" - "fmt" - - "errors" - - "github.com/bytedance/gopkg/util/logger" - - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" - "github.com/gnmic/operator/internal/controller/discovery/core" - "sigs.k8s.io/controller-runtime/pkg/log" -) - -// this file implements the logic receive target updates via HTTP push -// REST API defined internal/apiserver - -// Loader implements the HTTP pull discovery mechanism -type Loader struct { - cfg core.LoaderConfig -} - -// New returns a new http_push loader instance configured with cfg -func New(cfg core.LoaderConfig) core.Loader { - return &Loader{cfg: cfg} -} - -func (l *Loader) Name() string { - return "http_push" -} - -func (l *Loader) Start( - ctx context.Context, - targetsourceName string, - spec gnmicv1alpha1.TargetSourceSpec, - out chan<- []core.DiscoveryMessage, -) error { - logger := log.FromContext(ctx).WithValues( - "component", "loader", - "name", l.Name(), - "targetsource", targetsourceName, - ) - logger.Info("HTTP push loader started") - - // Input Validation of spec - if spec.Provider == nil || spec.Provider.HTTP == nil { - return errors.New("http_push loader requires spec.provider.http to be set") - } - - // Receive target updates via HTTP push - var targetEvents []core.DiscoveryEvent - - if err := core.SendEvents(ctx, out, targetEvents, l.cfg.ChunkSize); err != nil { - logger.Error(err, "failed to send events") - return nil - } - return nil -} - -func SendTargetToLoader(dm []core.DiscoveryMessage) { - logger.Info("SendTargetToLoader %s", dm) - // for _, target := range payload { - // if target.Name != nil { - // fmt.Printf("name: %s, ", *target.Name) - // } - // if target.Address != nil { - // fmt.Printf("address: %s, ", *target.Address) - // } - // if target.Profile != nil { - // fmt.Printf("profile: %s, ", *target.Profile) - // } - // if target.Tags != nil { - // fmt.Printf("tags: %s", *target.Tags) - // } - fmt.Printf("SentTargetToLoader called") - //} -} diff --git a/internal/controller/discovery/loaders/push/loader_test.go b/internal/controller/discovery/loaders/push/loader_test.go deleted file mode 100644 index 63fdf61..0000000 --- a/internal/controller/discovery/loaders/push/loader_test.go +++ /dev/null @@ -1 +0,0 @@ -package push From 32cb07705b74192e4f66d69e28b28b42ded63b87 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 23 Apr 2026 15:55:17 +0000 Subject: [PATCH 030/110] add core.SendEvents, ShouldBindJSON doesn't work --- cmd/main.go | 3 +- go.mod | 2 +- internal/apiserver/apiserver.go | 54 ++++++++++++++----- internal/apiserver/gen.go | 50 +++++++++++++---- internal/apiserver/openapi.yaml | 29 +++++++--- .../resources/targetsources/ctestPull.yaml | 12 ----- 6 files changed, 105 insertions(+), 45 deletions(-) delete mode 100644 lab/dev/resources/targetsources/ctestPull.yaml diff --git a/cmd/main.go b/cmd/main.go index 225f750..03d48d7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -233,10 +233,11 @@ func main() { if apiAddr != "" { api, err := apiserver.New(apiAddr, clusterReconciler) - if err != nil { + if err != nil { setupLog.Error(err, "unable to intialize gin API server") os.Exit(1) } + api.ChunkSize = discoveryChunkSize api.DiscoveryRegistry = discoveryRegistry err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error) diff --git a/go.mod b/go.mod index 507554e..0a1d66c 100644 --- a/go.mod +++ b/go.mod @@ -87,7 +87,7 @@ require ( github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect - github.com/google/uuid v1.6.0 // indirect + github.com/google/uuid v1.6.0 github.com/json-iterator/go v1.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index e112560..0aed77f 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -4,6 +4,8 @@ package apiserver // or use go generate ./internal/apiserver in the console (install from https://github.com/oapi-codegen/oapi-codegen) import ( + "context" + "fmt" "net/http" "github.com/gin-gonic/gin" @@ -16,8 +18,8 @@ type APIServer struct { Server *http.Server router *gin.Engine clusterReconciler *controller.ClusterReconciler - - DiscoveryRegistry *registry.Registry[[]core.DiscoveryMessage] + ChunkSize int + DiscoveryRegistry *registry.Registry[[]core.DiscoveryMessage] // change to lowercase? } func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServer, error) { @@ -50,21 +52,47 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { // CreateTargets binds payload to Target struct defined in openapi.yaml and sends it to pull loader func (a *APIServer) CreateTargets(c *gin.Context) { - var payload []Target - if err := c.ShouldBindJSON(&payload); err != nil { + // logger.Info("Create Targets called") + var payloadTarget []Target + var payloadTargetSource TargetSource + fmt.Println("Binding Target to PayloadTarget") + if err := c.ShouldBindJSON(&payloadTarget); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + fmt.Printf("err: %s", err.Error) + return + } + fmt.Printf("payloadTarget: %s", payloadTarget) + if err := c.ShouldBindJSON(&payloadTargetSource); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - targets := []core.DiscoveredTarget{} - for _, target := range payload { - targets = append(targets, core.DiscoveredTarget{ - Name: *target.Name, - Address: *target.Address, - Labels: map[string]string{"key": "Is this a tag?"}, + + targets := []core.DiscoveryEvent{} + for _, target := range payloadTarget { + event := core.CREATE + switch *target.Operation { + case Create: + event = core.CREATE + case Delete: + event = core.DELETE + } + targets = append(targets, core.DiscoveryEvent{ + Target: core.DiscoveredTarget{ + Name: *target.Name, + Address: *target.Address, + Labels: map[string]string{"key": "Is this a tag?"}, + }, + Event: event, }) } - // discovery / core / helpers / sendEvents to send received udpates to TagetManager - // loader push not needed - c.JSON(http.StatusOK, payload) + ch, ok := a.DiscoveryRegistry.Get(*payloadTargetSource.Namespace + "/" + *payloadTargetSource.Name) + if !ok { + // Error message to be udpated!! + c.JSON(http.StatusBadRequest, gin.H{"error": "Target Source doesn't exist"}) + return + } + + core.SendEvents(context.Background(), ch, targets, a.ChunkSize) + c.JSON(http.StatusOK, payloadTarget) } diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index 88ed5ab..acc383b 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -16,12 +16,40 @@ import ( "github.com/gin-gonic/gin" ) +// Defines values for TargetOperation. +const ( + Create TargetOperation = "create" + Delete TargetOperation = "delete" +) + +// Valid indicates whether the value is a known member of the TargetOperation enum. +func (e TargetOperation) Valid() bool { + switch e { + case Create: + return true + case Delete: + return true + default: + return false + } +} + // Target defines model for Target. type Target struct { - Address *string `json:"address,omitempty"` - Name *string `json:"name,omitempty"` - Profile *string `json:"profile,omitempty"` - Tags *[]string `json:"tags,omitempty"` + Address *string `json:"address,omitempty"` + Name *string `json:"name,omitempty"` + Operation *TargetOperation `json:"operation,omitempty"` + Profile *string `json:"profile,omitempty"` + Tags *[]string `json:"tags,omitempty"` +} + +// TargetOperation defines model for Target.Operation. +type TargetOperation string + +// TargetSource defines model for TargetSource. +type TargetSource struct { + Name *string `json:"name,omitempty"` + Namespace *string `json:"namespace,omitempty"` } // ServerInterface represents all server handlers. @@ -103,12 +131,14 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/2ySwW7bMAyGX0XgdjRsJ7v5NgRDkMO2YMsLqDLtKLAlgaQLBIHevZDsNnWTkwmKP/nz", - "o29g/Bi8QycMzQ3YnHHUOTxp6lFSFMgHJLGY87ptCTmHcg0IDbCQdT3EApwe8elDIN/Z4fmb6D53s4Lj", - "87ZLQhPpK8R7wr9c0AjElLKu81lsJc2B/s/vg1F/A5IWT+rfr/8n9fN4gAJekdh6Bw3UZV1u0gAf0Olg", - "oYEf5basoYCg5ZzNVIZQC840ciZ4zlh87m29O7TQwG5VVgAhB+94ZratN+ljvBN0WaxDGKzJ8urCyc07", - "/BWK74QdNPCtup+pWm5ULQd65BMLaJEN2SDznosrNa/SKp6MQeZuGoaZJ0/jqOn6sYaSRWGdkjOqNcws", - "qcKgs+vlJ1nT2KPshokF6ZjKHnDU6bM2+aleEcpEDtsv5vYoysxlKo+PMca3AAAA//9/7qb/wwIAAA==", + "H4sIAAAAAAAC/4RSTW/bMAz9KwK3oxG73c23oRiKHLYVa2/DDprMJCpsSSOpAUHg/z7oY0k8Z+hJBPlI", + "ke+9Exg/Be/QCUN/AjYHnHQOXzTtUVIUyAcksZjzehgIOYdyDAg9sJB1e5gbcHrCm4U0QIv1LlXRxQn6", + "72AItSA0MOCIgvCjWTcG8js73h4qep/XsILT7X1qQhPpI8yXhP/5ikYSolz57CMZXN/633tSgYM2t6rr", + "b1LKup3PYCvpHNh/+bw16mvmxZP69un5RX182kIDv5E4MwXdptvcVfqcDhZ6+LC533TQQNByyCu2hcVy", + "R84Ez1m2M+fbAXp4WMAaIOTgHZc777u79BjvBF1u1iGM1uT29pWLbsUca5YuTjkr8Z5wBz28ay/2aqu3", + "2gpfybMW4+0pFZsIJvwVLeGQnLUo/p175a9rYQZkQzYUc1Yoq8LqoDgag8y7OI7FQRynSdPxzKiS2mGd", + "kgOqpa65pQ2jzgRWlpbCPKI8jJEF6SnBVsp06VkueYVXhBLJ4fDPco8oyhSYyt/P8zz/CQAA//+Shfmd", + "7gMAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index fccf33e..bd66bb8 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -20,12 +20,24 @@ paths: content: application/json: schema: - type: array - items: - $ref: '#/components/schemas/Target' - + type: object + required: [TargetSource, Target] + properties: + TargetSource: + $ref: '#/components/schemas/TargetSource' + Target: + type: array + items: + $ref: '#/components/schemas/Target' components: schemas: + TargetSource: + type: object + properties: + namespace: + type: string + name: + type: string Target: type: object properties: @@ -39,7 +51,8 @@ components: type: array items: type: string - # username: - # type: string - # password: # not sure if the password is needed - # type: string \ No newline at end of file + operation: + type: string + enum: + - create + - delete \ No newline at end of file diff --git a/lab/dev/resources/targetsources/ctestPull.yaml b/lab/dev/resources/targetsources/ctestPull.yaml deleted file mode 100644 index 118e729..0000000 --- a/lab/dev/resources/targetsources/ctestPull.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: pull-loader -spec: - provider: - pull: - url: http://inventory-service:8080/targets - labels: - source: inventory - type: pull - profile: eos \ No newline at end of file From 8b4f5c9c840a0d01b8350c2ef698f4fdfb8619d2 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 23 Apr 2026 16:20:37 +0000 Subject: [PATCH 031/110] tests with ShouldBindBodyJSON --- internal/apiserver/apiserver.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 0aed77f..7f125cf 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -9,6 +9,7 @@ import ( "net/http" "github.com/gin-gonic/gin" + "github.com/gin-gonic/gin/binding" "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/registry" @@ -53,20 +54,24 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { // CreateTargets binds payload to Target struct defined in openapi.yaml and sends it to pull loader func (a *APIServer) CreateTargets(c *gin.Context) { // logger.Info("Create Targets called") + var payloadTarget []Target var payloadTargetSource TargetSource fmt.Println("Binding Target to PayloadTarget") - if err := c.ShouldBindJSON(&payloadTarget); err != nil { + // https://gin-gonic.com/en/docs/binding/bind-body-into-different-structs/ + if err := c.ShouldBindBodyWithJSON(&payloadTarget); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) fmt.Printf("err: %s", err.Error) return } fmt.Printf("payloadTarget: %s", payloadTarget) - if err := c.ShouldBindJSON(&payloadTargetSource); err != nil { + if err := c.ShouldBindBodyWithJSON(&payloadTargetSource); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } + // error {"error":"json: cannot unmarshal object into Go value of type []apiserver.Target"} + targets := []core.DiscoveryEvent{} for _, target := range payloadTarget { event := core.CREATE From 22683f4e4b0ee7853f45c8fce20c7d1646317162 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 23 Apr 2026 16:37:08 +0000 Subject: [PATCH 032/110] remove unused event action from DiscoverySnapshot --- internal/controller/discovery/core/types.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 69a407e..61209fd 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -27,7 +27,6 @@ type DiscoveryEvent struct { type DiscoverySnapshot struct { Targets []DiscoveredTarget - Event EventAction SnapshotID string IsLastChunk bool } From 922bbc6a6be0900f27e9aed9c09d6bce1c19caf6 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 07:32:40 +0000 Subject: [PATCH 033/110] rename target manager to target applier --- .../controller/discovery/target_applier.go | 18 +++++++++--------- internal/controller/targetsource_controller.go | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_applier.go index 153723c..3babebf 100644 --- a/internal/controller/discovery/target_applier.go +++ b/internal/controller/discovery/target_applier.go @@ -12,8 +12,8 @@ import ( "github.com/go-logr/logr" ) -// TargetManager consumes discovered targets and applies them to Kubernetes -type TargetManager struct { +// TargetApplier consumes discovered targets and applies them to Kubernetes +type TargetApplier struct { client client.Client scheme *runtime.Scheme targetSource *gnmicv1alpha1.TargetSource @@ -21,9 +21,9 @@ type TargetManager struct { collected map[string][]core.DiscoveredTarget } -// NewTargetManager wires a TargetManager instance -func NewTargetManager(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetManager { - return &TargetManager{ +// NewTargetApplier wires a TargetApplier instance +func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetApplier { + return &TargetApplier{ client: c, scheme: s, targetSource: ts, @@ -34,16 +34,16 @@ func NewTargetManager(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ // Run is a long‑running loop that receives target snapshots // and reconciles Target CRs accordingly -func (m *TargetManager) Run(ctx context.Context) error { +func (m *TargetApplier) Run(ctx context.Context) error { logger := log.FromContext(ctx). WithValues("targetSource", m.targetSource) - logger.Info("target manager started") + logger.Info("target applier started") for { select { case <-ctx.Done(): - logger.Info("target manager stopped") + logger.Info("target applier stopped") return nil case messages := <-m.in: @@ -83,7 +83,7 @@ func (m *TargetManager) Run(ctx context.Context) error { } // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly -func (m *TargetManager) processSnapshot(snapshotID string, logger logr.Logger) { +func (m *TargetApplier) processSnapshot(snapshotID string, logger logr.Logger) { targets := m.collected[snapshotID] delete(m.collected, snapshotID) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index c714acc..78d64d0 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -176,8 +176,8 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, ta // Start loader go loader.Start(runtimeCtx, targetSource.Name, targetSource.Spec, targetChannel) - // Start target manager - manager := discovery.NewTargetManager( + // Start target applier + manager := discovery.NewTargetApplier( r.Client, r.Scheme, targetSource, From 733927fa680c2896c83ee2863f7d2c2b24575448 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 07:51:37 +0000 Subject: [PATCH 034/110] implement key for registry as a comparable --- cmd/main.go | 3 +- internal/apiserver/apiserver.go | 3 +- .../controller/discovery/registry/registry.go | 39 +++++-------------- .../controller/targetsource_controller.go | 13 +++---- 4 files changed, 19 insertions(+), 39 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 5cf8169..e4bad31 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -28,6 +28,7 @@ import ( certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" @@ -85,7 +86,7 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - discoveryRegistry := registry.NewRegistry[[]core.DiscoveryMessage]() + discoveryRegistry := registry.NewRegistry[types.NamespacedName, []core.DiscoveryMessage]() mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index b84eb9a..17e5c82 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -7,13 +7,14 @@ import ( "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/registry" + "k8s.io/apimachinery/pkg/types" ) type APIServer struct { Server *http.Server clusterReconciler *controller.ClusterReconciler - DiscoveryRegistry *registry.Registry[[]core.DiscoveryMessage] + DiscoveryRegistry *registry.Registry[types.NamespacedName, []core.DiscoveryMessage] } func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer { diff --git a/internal/controller/discovery/registry/registry.go b/internal/controller/discovery/registry/registry.go index 7da0757..1892d28e 100644 --- a/internal/controller/discovery/registry/registry.go +++ b/internal/controller/discovery/registry/registry.go @@ -5,39 +5,18 @@ import ( "sync" ) -/* USAGE - -// create registry once in main.go -discoveryReg := discovery.NewRegistry[[]core.DiscoveryMessage]() - -// inside targetsource controller, when starting discovery pipeline: -key := fmt.Sprintf("%s/%s", spec.Namespace, targetsourceName) -if err := discoveryReg.Register(key, out); err != nil { - logger.Error(err, "could not register loader") - return err -} -defer discoveryReg.Unregister(key) - -// CHECK REGISTRY -ch, ok := discoveryReg.Get(ns + "/" + ts) -if !ok { - http.Error(w, "no loader for targetsource", http.StatusNotFound) - return -} -// then deliver payload to ch -*/ - -// Registry is a thread-safe map: key -> channel of T. -type Registry[T any] struct { +// Registry is a thread-safe key -> channel registry +// K must be comparable so it can be used as a map key +type Registry[K comparable, V any] struct { mu sync.RWMutex - m map[string]chan<- T + m map[K]chan<- V } -func NewRegistry[T any]() *Registry[T] { - return &Registry[T]{m: make(map[string]chan<- T)} +func NewRegistry[K comparable, V any]() *Registry[K, V] { + return &Registry[K, V]{m: make(map[K]chan<- V)} } -func (r *Registry[T]) Register(key string, ch chan<- T) error { +func (r *Registry[K, V]) Register(key K, ch chan<- V) error { r.mu.Lock() defer r.mu.Unlock() if _, exists := r.m[key]; exists { @@ -47,13 +26,13 @@ func (r *Registry[T]) Register(key string, ch chan<- T) error { return nil } -func (r *Registry[T]) Unregister(key string) { +func (r *Registry[K, V]) Unregister(key K) { r.mu.Lock() delete(r.m, key) r.mu.Unlock() } -func (r *Registry[T]) Get(key string) (chan<- T, bool) { +func (r *Registry[K, V]) Get(key K) (chan<- V, bool) { r.mu.RLock() ch, ok := r.m[key] r.mu.RUnlock() diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 78d64d0..d97b3a6 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -21,6 +21,7 @@ import ( "sync" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -50,7 +51,7 @@ type TargetSourceReconciler struct { BufferSize int ChunkSize int - DiscoveryRegistry *registry.Registry[[]core.DiscoveryMessage] + DiscoveryRegistry *registry.Registry[types.NamespacedName, []core.DiscoveryMessage] } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete @@ -149,7 +150,7 @@ func (r *TargetSourceReconciler) isPipelineRunning(key client.ObjectKey) bool { } // startDiscoveryPipeline creates and starts the loader and target manager -func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, targetSource *gnmicv1alpha1.TargetSource) error { +func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) error { cfg := core.LoaderConfig{ ChunkSize: r.ChunkSize, } @@ -167,8 +168,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, ta runtimeCtx, cancel := context.WithCancel(context.Background()) targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) - registryKey := key.Namespace + "/" + key.Name - if err := r.DiscoveryRegistry.Register(registryKey, targetChannel); err != nil { + if err := r.DiscoveryRegistry.Register(key, targetChannel); err != nil { cancel() return err } @@ -194,7 +194,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key client.ObjectKey, ta // stopDiscovery stops and removes a running discovery pipeline // for the given TargetSource key -func (r *TargetSourceReconciler) stopDiscovery(key client.ObjectKey) { +func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) { r.mu.Lock() running, ok := r.running[key] if ok { @@ -204,8 +204,7 @@ func (r *TargetSourceReconciler) stopDiscovery(key client.ObjectKey) { r.mu.Unlock() if ok { - registryKey := key.Namespace + "/" + key.Name - r.DiscoveryRegistry.Unregister(registryKey) + r.DiscoveryRegistry.Unregister(key) } } From 9d305601d18ae0f9f4d9f0168ec799b15e8b4a2a Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 08:10:41 +0000 Subject: [PATCH 035/110] fix error message and add a word of caution for key comparables --- internal/controller/discovery/registry/registry.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/controller/discovery/registry/registry.go b/internal/controller/discovery/registry/registry.go index 1892d28e..093bd2c 100644 --- a/internal/controller/discovery/registry/registry.go +++ b/internal/controller/discovery/registry/registry.go @@ -7,6 +7,7 @@ import ( // Registry is a thread-safe key -> channel registry // K must be comparable so it can be used as a map key +// DO NOT USE a pointer type as K type Registry[K comparable, V any] struct { mu sync.RWMutex m map[K]chan<- V @@ -20,7 +21,7 @@ func (r *Registry[K, V]) Register(key K, ch chan<- V) error { r.mu.Lock() defer r.mu.Unlock() if _, exists := r.m[key]; exists { - return fmt.Errorf("already registered: %s", key) + return fmt.Errorf("already registered: %v", key) } r.m[key] = ch return nil From dafa82bb1fd1fbbb5369d14ff82594be38b19ddb Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 08:11:17 +0000 Subject: [PATCH 036/110] consistently use namespaced name as refference to the targetsource --- .../discovery/core/loader_interface.go | 3 ++- internal/controller/discovery/loader.go | 8 ++++---- .../controller/discovery/loaders/http/loader.go | 11 ++++++----- internal/controller/targetsource_controller.go | 17 ++++++++--------- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/internal/controller/discovery/core/loader_interface.go b/internal/controller/discovery/core/loader_interface.go index 17cd5f4..8964be8 100644 --- a/internal/controller/discovery/core/loader_interface.go +++ b/internal/controller/discovery/core/loader_interface.go @@ -4,6 +4,7 @@ import ( "context" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "k8s.io/apimachinery/pkg/types" ) // Loader defines a pluggable TargetSource loader interface @@ -16,7 +17,7 @@ type Loader interface { // The loader must stop cleanly when ctx is cancelled Start( ctx context.Context, - targetsourceName string, + targetsourceName types.NamespacedName, spec gnmicv1alpha1.TargetSourceSpec, out chan<- []DiscoveryMessage, ) error diff --git a/internal/controller/discovery/loader.go b/internal/controller/discovery/loader.go index 42ce8da..0d8ddd3 100644 --- a/internal/controller/discovery/loader.go +++ b/internal/controller/discovery/loader.go @@ -6,19 +6,19 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" http "github.com/gnmic/operator/internal/controller/discovery/loaders/http" + "k8s.io/apimachinery/pkg/types" ) // NewLoader creates a loader by name -func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpec, cfg core.LoaderConfig) (core.Loader, error) { - loaderName := namespace + "/" + name +func NewLoader(name types.NamespacedName, spec gnmicv1alpha1.TargetSourceSpec, cfg core.LoaderConfig) (core.Loader, error) { switch { case spec.Provider.HTTP != nil: return http.New(cfg), nil case spec.Provider.Consul != nil: - return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName) + return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", name) default: - return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName) + return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", name) } } diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index f014a2f..09bb7d6 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/log" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" @@ -27,14 +28,14 @@ func (l *Loader) Name() string { func (l *Loader) Start( ctx context.Context, - targetsourceName string, + targetsourceNN types.NamespacedName, spec gnmicv1alpha1.TargetSourceSpec, out chan<- []core.DiscoveryMessage, ) error { logger := log.FromContext(ctx).WithValues( "component", "loader", "name", l.Name(), - "targetsource", targetsourceName, + "targetsource", targetsourceNN, ) logger.Info("HTTP loader started") @@ -51,17 +52,17 @@ func (l *Loader) Start( case <-ticker.C: // Example snapshot (placeholder) - snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceName, uuid.NewString()) + snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceNN, uuid.NewString()) targets := []core.DiscoveredTarget{ { Name: "ceos1", Address: "clab-3-nodes-ceos1:6030", - Labels: map[string]string{"TargetSource": targetsourceName}, + Labels: map[string]string{"TargetSource": targetsourceNN.String()}, }, { Name: "leaf1", Address: "clab-3-nodes-leaf1:57400", - Labels: map[string]string{"TargetSource": targetsourceName}, + Labels: map[string]string{"TargetSource": targetsourceNN.String()}, }, } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index d97b3a6..62b057d 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -46,7 +46,7 @@ type TargetSourceReconciler struct { Scheme *runtime.Scheme mu sync.Mutex - running map[client.ObjectKey]runningSource + running map[types.NamespacedName]runningSource BufferSize int ChunkSize int @@ -96,7 +96,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request } // getTargetSource retrieves a TargetSource by name, handling cleanup if not found -func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key client.ObjectKey) (*gnmicv1alpha1.TargetSource, error) { +func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key types.NamespacedName) (*gnmicv1alpha1.TargetSource, error) { var targetSource gnmicv1alpha1.TargetSource if err := r.Get(ctx, key, &targetSource); err != nil { // If the TargetSource no longer exists, ensure runtime cleanup @@ -109,9 +109,9 @@ func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key client } // handleTargetSourceDeletion stops the discovery pipeline and removes the finalizer -func (r *TargetSourceReconciler) handleTargetSourceDeletion(ctx context.Context, key client.ObjectKey, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { +func (r *TargetSourceReconciler) handleTargetSourceDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { logger := log.FromContext(ctx) - logger.Info("TargetSource is being deleted, stopping pipeline", "name", targetSource.Name) + logger.Info("TargetSource is being deleted, stopping pipeline", "name", key) r.stopDiscovery(key) @@ -141,7 +141,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour } // isPipelineRunning checks if a discovery pipeline is already running for the given key -func (r *TargetSourceReconciler) isPipelineRunning(key client.ObjectKey) bool { +func (r *TargetSourceReconciler) isPipelineRunning(key types.NamespacedName) bool { r.mu.Lock() defer r.mu.Unlock() @@ -156,8 +156,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName } loader, err := discovery.NewLoader( - targetSource.ObjectMeta.Name, - targetSource.ObjectMeta.Namespace, + key, targetSource.Spec, cfg, ) @@ -174,7 +173,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName } // Start loader - go loader.Start(runtimeCtx, targetSource.Name, targetSource.Spec, targetChannel) + go loader.Start(runtimeCtx, key, targetSource.Spec, targetChannel) // Start target applier manager := discovery.NewTargetApplier( @@ -210,7 +209,7 @@ func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) { // SetupWithManager sets up the controller with the Manager. func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { - r.running = make(map[client.ObjectKey]runningSource) + r.running = make(map[types.NamespacedName]runningSource) return ctrl.NewControllerManagedBy(mgr). For(&gnmicv1alpha1.TargetSource{}). From 2973c03a665beeb3b53ef7ff71d55921c21053e1 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 09:05:21 +0000 Subject: [PATCH 037/110] improve context cancling and error handling --- .../controller/discovery/target_applier.go | 102 ++++++++++++------ 1 file changed, 68 insertions(+), 34 deletions(-) diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_applier.go index 3babebf..7fed5c9 100644 --- a/internal/controller/discovery/target_applier.go +++ b/internal/controller/discovery/target_applier.go @@ -37,49 +37,83 @@ func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ func (m *TargetApplier) Run(ctx context.Context) error { logger := log.FromContext(ctx). WithValues("targetSource", m.targetSource) - logger.Info("target applier started") - for { + queue := make([]core.DiscoveryMessage, 0, 265) + + for ctx.Err() == nil { select { + case batch, ok := <-m.in: + if !ok { + // Channel closed, pipeline is shutting down + logger.Info("input channel closed, stopping target applier") + return nil + } + queue = append(queue, batch...) + case <-ctx.Done(): - logger.Info("target applier stopped") + logger.Info("context canceled, stopping target applier") return nil + } - case messages := <-m.in: - for _, message := range messages { - // Type assert to determine if this is a snapshot or event - switch msg := message.(type) { - case core.DiscoverySnapshot: - // Collect snapshot chunks - logger.Info( - "received snapshot chunk", - "snapshotID", msg.SnapshotID, - "targetCount", len(msg.Targets), - ) - m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Targets...) - if msg.IsLastChunk { - m.processSnapshot(msg.SnapshotID, logger) - } - - case core.DiscoveryEvent: - // Process individual event-driven update - logger.Info( - "received discovery event", - "target", msg.Target.Name, - ) - switch msg.Event { - case core.CREATE: - logger.Info("Would create target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) - case core.UPDATE: - logger.Info("Would update target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) - case core.DELETE: - logger.Info("Would delete target", "name", msg.Target.Name) - } - } + for len(queue) > 0 { + if ctx.Err() != nil { + break } + + msg := queue[0] + queue = queue[1:] + + if err := m.handleMessage(ctx, msg, logger); err != nil { + // Returning error lets the supervisor (controller) + // tear down and restart the pipeline via reconciliation + // Q: when to return an error vs just log and continue? + return err + } + } } + + logger.Info("target applier stopped") + return nil +} + +func (m *TargetApplier) handleMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { + if err := ctx.Err(); err != nil { + return err + } + + // Type assert to determine if this is a snapshot or event + switch msg := message.(type) { + case core.DiscoverySnapshot: + // Collect snapshot chunks + logger.Info( + "received snapshot chunk", + "snapshotID", msg.SnapshotID, + "targetCount", len(msg.Targets), + ) + m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Targets...) + if msg.IsLastChunk { + m.processSnapshot(msg.SnapshotID, logger) + } + + case core.DiscoveryEvent: + // Process individual event-driven update + logger.Info( + "received discovery event", + "target", msg.Target.Name, + ) + switch msg.Event { + case core.CREATE: + logger.Info("Would create target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) + case core.UPDATE: + logger.Info("Would update target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) + case core.DELETE: + logger.Info("Would delete target", "name", msg.Target.Name) + } + } + + return nil } // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly From c95bdaf389038386a0b0b98759c98d4c10cb3f31 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 10:01:31 +0000 Subject: [PATCH 038/110] add supervised goroutines --- .../controller/targetsource_controller.go | 54 +++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 62b057d..9fad373 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -18,6 +18,7 @@ package controller import ( "context" + "fmt" "sync" "k8s.io/apimachinery/pkg/runtime" @@ -172,17 +173,45 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName return err } + // goroutines use done channel to report termination (nil or error) back to supervisor + // Buffer size = supervised goroutines = 2 (loader + applier) + done := make(chan error, 2) + // Start loader - go loader.Start(runtimeCtx, key, targetSource.Spec, targetChannel) + go runWithRecovery( + runtimeCtx, + "loader", + func(ctx context.Context) error { + return loader.Start(ctx, key, targetSource.Spec, targetChannel) + }, + done, + ) // Start target applier - manager := discovery.NewTargetApplier( + applier := discovery.NewTargetApplier( r.Client, r.Scheme, targetSource, targetChannel, ) - go manager.Run(runtimeCtx) + go runWithRecovery( + runtimeCtx, + "target-applier", + applier.Run, + done, + ) + + // Supervision goroutine to handle pipeline termination + go func() { + err := <-done + logger := log.FromContext(context.Background()).WithValues("targetSource", key) + if err != nil { + logger.Error(err, "Discovery pipeline terminated with error") + } + + // Ensure cleanup on termination + r.stopDiscovery(key) + }() r.mu.Lock() r.running[key] = runningSource{cancel: cancel} @@ -207,6 +236,25 @@ func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) { } } +// runWithRecovery executes a worker function under panic protection +// and reports termination (nil or error) through done. +func runWithRecovery( + ctx context.Context, + name string, + run func(context.Context) error, + done chan<- error, +) { + defer func() { + if r := recover(); r != nil { + done <- fmt.Errorf("panic in %s: %v", name, r) + } + }() + + // Normal exit path + err := run(ctx) + done <- err +} + // SetupWithManager sets up the controller with the Manager. func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { r.running = make(map[types.NamespacedName]runningSource) From 0aa883d98c940ebf374c6b9492522e63a601ac6d Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 12:52:08 +0000 Subject: [PATCH 039/110] refactor target applier --- internal/controller/discovery/target_applier.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_applier.go index 7fed5c9..c60f2b8 100644 --- a/internal/controller/discovery/target_applier.go +++ b/internal/controller/discovery/target_applier.go @@ -36,10 +36,14 @@ func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ // and reconciles Target CRs accordingly func (m *TargetApplier) Run(ctx context.Context) error { logger := log.FromContext(ctx). - WithValues("targetSource", m.targetSource) + WithValues( + "name", m.targetSource.Name, + "namespace", m.targetSource.Namespace, + ) + logger.Info("target applier started") - queue := make([]core.DiscoveryMessage, 0, 265) + queue := []core.DiscoveryMessage{} for ctx.Err() == nil { select { @@ -58,7 +62,7 @@ func (m *TargetApplier) Run(ctx context.Context) error { for len(queue) > 0 { if ctx.Err() != nil { - break + return ctx.Err() } msg := queue[0] From 27b2b1f711a4f60edd2609d8e2822adbfaf07991 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 14:15:34 +0000 Subject: [PATCH 040/110] add supervisor for the discovery pipelines --- internal/controller/discovery/supervisor.go | 123 ++++++++++++++++++ .../controller/targetsource_controller.go | 117 +++++++---------- 2 files changed, 171 insertions(+), 69 deletions(-) create mode 100644 internal/controller/discovery/supervisor.go diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go new file mode 100644 index 0000000..ff19604 --- /dev/null +++ b/internal/controller/discovery/supervisor.go @@ -0,0 +1,123 @@ +package discovery + +import ( + "context" + "sync" + "time" + + "sigs.k8s.io/controller-runtime/pkg/log" +) + +type ComponentExit struct { + Name string + Err error +} + +type RestartPolicy struct { + MaxRestarts int + Backoff time.Duration +} + +type Supervisor struct { + ctx context.Context + cancel context.CancelFunc + policy RestartPolicy + failures int + exits chan ComponentExit + wg sync.WaitGroup + stopped bool + stopMu sync.Mutex +} + +func NewSupervisor(parentCtx context.Context, policy RestartPolicy) *Supervisor { + ctx, cancel := context.WithCancel(parentCtx) + return &Supervisor{ + ctx: ctx, + cancel: cancel, + policy: policy, + exits: make(chan ComponentExit, 4), + failures: 0, + } +} + +func (s *Supervisor) Context() context.Context { + return s.ctx +} + +func (s *Supervisor) Stop() { + s.stopMu.Lock() + defer s.stopMu.Unlock() + + if s.stopped { + return + } + + s.stopped = true + s.cancel() +} + +func (s *Supervisor) Run( + start func(ctx context.Context, exits chan<- ComponentExit), +) error { + logger := log.FromContext(s.ctx).WithName("discovery-supervisor") + + for { + if s.failures > 0 { + logger.Info("Restarting pipeline", + "attempt", s.failures, + "maxAttempts", s.policy.MaxRestarts, + ) + + runtimeCtx, cancel := context.WithCancel(s.ctx) + s.wg = sync.WaitGroup{} + start(runtimeCtx, s.exits) + exit := <-s.exits // first failure wins + + logger.Error(exit.Err, + "Pipeline component crashed", + "component", exit.Name, + ) + + cancel() + s.wg.Wait() + + s.failures++ + if s.failures >= s.policy.MaxRestarts { + logger.Error(exit.Err, + "Pipeline exceeded maximum restart attempts; waiting for next reconciliation to restart", + "restarts", s.failures, + ) + s.Stop() + return exit.Err + } + + select { + case <-time.After(s.policy.Backoff): + // continue to restart + case <-s.ctx.Done(): + // Supervisor context canceled during backoff + return s.ctx.Err() + } + } + } +} + +func (s *Supervisor) Go(name string, fn func(ctx context.Context) error) { + s.wg.Add(1) + + go func() { + defer s.wg.Done() + + err := fn(s.ctx) + if err == nil { + err = context.Canceled // treat normal exit as cancellation + } + + select { + case s.exits <- ComponentExit{Name: name, Err: err}: + // exit reported successfully + case <-s.ctx.Done(): + // Supervisor context canceled before reporting exit + } + }() +} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 9fad373..5d83db9 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -18,8 +18,8 @@ package controller import ( "context" - "fmt" "sync" + "time" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -33,9 +33,14 @@ import ( "github.com/gnmic/operator/internal/controller/discovery/core" _ "github.com/gnmic/operator/internal/controller/discovery/loaders/all" "github.com/gnmic/operator/internal/controller/discovery/registry" + "github.com/go-logr/logr" ) -const targetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer" +const ( + targetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer" + pipelineMaxRestarts = 5 + pipelineBackoff = 3 * time.Second +) type runningSource struct { cancel context.CancelFunc @@ -63,8 +68,8 @@ type TargetSourceReconciler struct { // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx).WithValues( - "Name", req.NamespacedName, + logger := log.FromContext(ctx).WithName("targetsource controller").WithValues( + "targetsource", req.NamespacedName, ) targetSource, err := r.getTargetSource(ctx, req.NamespacedName) @@ -88,7 +93,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request } // Start discovery pipeline - if err := r.startDiscoveryPipeline(req.NamespacedName, targetSource); err != nil { + if err := r.startDiscoveryPipeline(req.NamespacedName, targetSource, logger); err != nil { return ctrl.Result{}, err } @@ -151,70 +156,63 @@ func (r *TargetSourceReconciler) isPipelineRunning(key types.NamespacedName) boo } // startDiscoveryPipeline creates and starts the loader and target manager -func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) error { - cfg := core.LoaderConfig{ - ChunkSize: r.ChunkSize, - } - - loader, err := discovery.NewLoader( - key, - targetSource.Spec, - cfg, +func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error { + supervisor := discovery.NewSupervisor( + context.Background(), + discovery.RestartPolicy{ + MaxRestarts: pipelineMaxRestarts, + Backoff: pipelineBackoff, + }, ) - if err != nil { - return err - } - runtimeCtx, cancel := context.WithCancel(context.Background()) targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) - if err := r.DiscoveryRegistry.Register(key, targetChannel); err != nil { - cancel() return err } - // goroutines use done channel to report termination (nil or error) back to supervisor - // Buffer size = supervised goroutines = 2 (loader + applier) - done := make(chan error, 2) + start := func(ctx context.Context, exits chan<- discovery.ComponentExit) { + // Create loader instance + loader, err := discovery.NewLoader( + key, + targetSource.Spec, + core.LoaderConfig{ + ChunkSize: r.ChunkSize, + }, + ) + if err != nil { + return + } + + // Create target applier instance + applier := discovery.NewTargetApplier( + r.Client, + r.Scheme, + targetSource, + targetChannel, + ) - // Start loader - go runWithRecovery( - runtimeCtx, - "loader", - func(ctx context.Context) error { + // Start loader + supervisor.Go("loader", func(ctx context.Context) error { return loader.Start(ctx, key, targetSource.Spec, targetChannel) - }, - done, - ) + }) + // Start target applier + supervisor.Go("target-applier", applier.Run) - // Start target applier - applier := discovery.NewTargetApplier( - r.Client, - r.Scheme, - targetSource, - targetChannel, - ) - go runWithRecovery( - runtimeCtx, - "target-applier", - applier.Run, - done, - ) + } - // Supervision goroutine to handle pipeline termination go func() { - err := <-done - logger := log.FromContext(context.Background()).WithValues("targetSource", key) + err := supervisor.Run(start) if err != nil { - logger.Error(err, "Discovery pipeline terminated with error") + logger.Error(err, "Discovery pipeline stopped permanently") } - // Ensure cleanup on termination + close(targetChannel) + r.DiscoveryRegistry.Unregister(key) r.stopDiscovery(key) }() r.mu.Lock() - r.running[key] = runningSource{cancel: cancel} + r.running[key] = runningSource{cancel: supervisor.Stop} r.mu.Unlock() return nil @@ -236,25 +234,6 @@ func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) { } } -// runWithRecovery executes a worker function under panic protection -// and reports termination (nil or error) through done. -func runWithRecovery( - ctx context.Context, - name string, - run func(context.Context) error, - done chan<- error, -) { - defer func() { - if r := recover(); r != nil { - done <- fmt.Errorf("panic in %s: %v", name, r) - } - }() - - // Normal exit path - err := run(ctx) - done <- err -} - // SetupWithManager sets up the controller with the Manager. func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { r.running = make(map[types.NamespacedName]runningSource) From 22fe2d894e2109c817a11b3153f298ba0fb8eb06 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 14:55:21 +0000 Subject: [PATCH 041/110] improve readability --- internal/controller/targetsource_controller.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 5d83db9..db60520 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -38,8 +38,9 @@ import ( const ( targetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer" - pipelineMaxRestarts = 5 - pipelineBackoff = 3 * time.Second + + pipelineMaxRestarts = 5 + pipelineBackoff = 3 * time.Second ) type runningSource struct { From 58538c76c0583e031b56031f72e639450c918910 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 15:02:48 +0000 Subject: [PATCH 042/110] remove side-effects from getter getTargetSource --- internal/controller/targetsource_controller.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index db60520..33342b4 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -75,6 +75,12 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request targetSource, err := r.getTargetSource(ctx, req.NamespacedName) if err != nil { + // If the TargetSource no longer exists, ensure runtime cleanup + if client.IgnoreNotFound(err) == nil { + logger.Info("TargetSource not found, ensuring cleanup") + r.stopDiscovery(req.NamespacedName) + return ctrl.Result{}, nil + } return ctrl.Result{}, err } @@ -106,11 +112,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key types.NamespacedName) (*gnmicv1alpha1.TargetSource, error) { var targetSource gnmicv1alpha1.TargetSource if err := r.Get(ctx, key, &targetSource); err != nil { - // If the TargetSource no longer exists, ensure runtime cleanup - if client.IgnoreNotFound(err) == nil { - r.stopDiscovery(key) - } - return nil, client.IgnoreNotFound(err) + return nil, err } return &targetSource, nil } From 4f0457ec86f4ed5df64a4216aadc8e3fc3551391 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 24 Apr 2026 17:38:41 +0000 Subject: [PATCH 043/110] redesign supervisor --- internal/controller/discovery/supervisor.go | 145 ++++++++---------- .../controller/targetsource_controller.go | 85 +++++----- 2 files changed, 106 insertions(+), 124 deletions(-) diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go index ff19604..c716965 100644 --- a/internal/controller/discovery/supervisor.go +++ b/internal/controller/discovery/supervisor.go @@ -8,116 +8,93 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) -type ComponentExit struct { - Name string - Err error -} - type RestartPolicy struct { MaxRestarts int Backoff time.Duration } +type Component struct { + Name string + Run func(ctx context.Context) error + Policy RestartPolicy +} + type Supervisor struct { - ctx context.Context - cancel context.CancelFunc - policy RestartPolicy - failures int - exits chan ComponentExit - wg sync.WaitGroup - stopped bool - stopMu sync.Mutex + ctx context.Context + cancel context.CancelFunc + + stopped bool + mu sync.Mutex + + components []Component } -func NewSupervisor(parentCtx context.Context, policy RestartPolicy) *Supervisor { - ctx, cancel := context.WithCancel(parentCtx) +func NewSupervisor(parent context.Context) *Supervisor { + ctx, cancel := context.WithCancel(parent) return &Supervisor{ - ctx: ctx, - cancel: cancel, - policy: policy, - exits: make(chan ComponentExit, 4), - failures: 0, + ctx: ctx, + cancel: cancel, } } -func (s *Supervisor) Context() context.Context { - return s.ctx +func (s *Supervisor) AddComponent(c Component) { + s.components = append(s.components, c) } -func (s *Supervisor) Stop() { - s.stopMu.Lock() - defer s.stopMu.Unlock() +func (s *Supervisor) runComponent(c Component) { + logger := log.FromContext(s.ctx).WithValues( + "component", c.Name, + ) - if s.stopped { - return - } - - s.stopped = true - s.cancel() -} - -func (s *Supervisor) Run( - start func(ctx context.Context, exits chan<- ComponentExit), -) error { - logger := log.FromContext(s.ctx).WithName("discovery-supervisor") + failures := 0 for { - if s.failures > 0 { - logger.Info("Restarting pipeline", - "attempt", s.failures, - "maxAttempts", s.policy.MaxRestarts, - ) + err := c.Run(s.ctx) + if s.ctx.Err() != nil { + return + } - runtimeCtx, cancel := context.WithCancel(s.ctx) - s.wg = sync.WaitGroup{} - start(runtimeCtx, s.exits) - exit := <-s.exits // first failure wins + failures++ + logger.Error(err, + "Component failed", + "attempt", failures, + ) - logger.Error(exit.Err, - "Pipeline component crashed", - "component", exit.Name, + if failures >= c.Policy.MaxRestarts { + logger.Error(err, + "Component exceeded restart limit; stopping discovery pipeline", + "restarts", failures, ) + s.Stop() + return + } - cancel() - s.wg.Wait() - - s.failures++ - if s.failures >= s.policy.MaxRestarts { - logger.Error(exit.Err, - "Pipeline exceeded maximum restart attempts; waiting for next reconciliation to restart", - "restarts", s.failures, - ) - s.Stop() - return exit.Err - } - - select { - case <-time.After(s.policy.Backoff): - // continue to restart - case <-s.ctx.Done(): - // Supervisor context canceled during backoff - return s.ctx.Err() - } + select { + case <-time.After(c.Policy.Backoff): + case <-s.ctx.Done(): + return } } } -func (s *Supervisor) Go(name string, fn func(ctx context.Context) error) { - s.wg.Add(1) +func (s *Supervisor) Run() { + for _, c := range s.components { + component := c + go s.runComponent(component) + } +} - go func() { - defer s.wg.Done() +func (s *Supervisor) Stop() { + s.mu.Lock() + defer s.mu.Unlock() - err := fn(s.ctx) - if err == nil { - err = context.Canceled // treat normal exit as cancellation - } + if s.stopped { + return + } + s.stopped = true + s.cancel() +} - select { - case s.exits <- ComponentExit{Name: name, Err: err}: - // exit reported successfully - case <-s.ctx.Done(): - // Supervisor context canceled before reporting exit - } - }() +func (s *Supervisor) Done() <-chan struct{} { + return s.ctx.Done() } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 33342b4..68f47eb 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -160,54 +160,57 @@ func (r *TargetSourceReconciler) isPipelineRunning(key types.NamespacedName) boo // startDiscoveryPipeline creates and starts the loader and target manager func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error { - supervisor := discovery.NewSupervisor( - context.Background(), - discovery.RestartPolicy{ - MaxRestarts: pipelineMaxRestarts, - Backoff: pipelineBackoff, - }, - ) + supervisor := discovery.NewSupervisor(context.Background()) targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) if err := r.DiscoveryRegistry.Register(key, targetChannel); err != nil { return err } - start := func(ctx context.Context, exits chan<- discovery.ComponentExit) { - // Create loader instance - loader, err := discovery.NewLoader( - key, - targetSource.Spec, - core.LoaderConfig{ - ChunkSize: r.ChunkSize, - }, - ) - if err != nil { - return - } + // Create loader instance + loader, err := discovery.NewLoader( + key, + targetSource.Spec, + core.LoaderConfig{ChunkSize: r.ChunkSize}, + ) + if err != nil { + return err + } - // Create target applier instance - applier := discovery.NewTargetApplier( - r.Client, - r.Scheme, - targetSource, - targetChannel, - ) + // Create target applier instance + applier := discovery.NewTargetApplier( + r.Client, + r.Scheme, + targetSource, + targetChannel, + ) - // Start loader - supervisor.Go("loader", func(ctx context.Context) error { + supervisor.AddComponent(discovery.Component{ + Name: "loader", + Run: func(ctx context.Context) error { return loader.Start(ctx, key, targetSource.Spec, targetChannel) - }) - // Start target applier - supervisor.Go("target-applier", applier.Run) + }, + Policy: discovery.RestartPolicy{ + MaxRestarts: pipelineMaxRestarts, + Backoff: pipelineBackoff, + }, + }) - } + supervisor.AddComponent(discovery.Component{ + Name: "target-applier", + Run: applier.Run, + Policy: discovery.RestartPolicy{ + MaxRestarts: pipelineMaxRestarts, + Backoff: pipelineBackoff, + }, + }) + + supervisor.Run() go func() { - err := supervisor.Run(start) - if err != nil { - logger.Error(err, "Discovery pipeline stopped permanently") - } + <-supervisor.Done() + + logger.Info("Pipeline stopped; performing final cleanup") close(targetChannel) r.DiscoveryRegistry.Unregister(key) @@ -215,25 +218,27 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName }() r.mu.Lock() - r.running[key] = runningSource{cancel: supervisor.Stop} + r.running[key] = runningSource{ + cancel: func() { + supervisor.Stop() + }, + } r.mu.Unlock() return nil } // stopDiscovery stops and removes a running discovery pipeline -// for the given TargetSource key func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) { r.mu.Lock() running, ok := r.running[key] if ok { - running.cancel() delete(r.running, key) } r.mu.Unlock() if ok { - r.DiscoveryRegistry.Unregister(key) + running.cancel() } } From 60491be6b980c081f46955c59a8dc995db26c2e0 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Sat, 25 Apr 2026 09:06:20 +0000 Subject: [PATCH 044/110] add dependency handling of discovery pipeline components --- api/v1alpha1/targetsource_types.go | 7 ++ api/v1alpha1/zz_generated.deepcopy.go | 21 ++++ .../operator.gnmic.dev_targetsources.yaml | 5 + internal/controller/discovery/supervisor.go | 112 +++++++++--------- .../controller/targetsource_controller.go | 67 +++++++---- 5 files changed, 133 insertions(+), 79 deletions(-) diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index feea000..a936e66 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -24,6 +24,8 @@ import ( // +kubebuilder:validation:Required type TargetSourceSpec struct { Provider *ProviderSpec `json:"provider"` + // +kubebuilder:validation:Optional + Webhook WebhookSpec `json:"webhook,omitempty"` // TargetLabels map[string]string `json:"targetLabels,omitempty"` @@ -37,6 +39,11 @@ type ProviderSpec struct { Consul *ConsulConfig `json:"consul,omitempty"` } +type WebhookSpec struct { + // +kubebuilder:validation:Optional + Enabled *bool `json:"enabled,omitempty"` +} + type HTTPConfig struct { // +kubebuilder:validation:MinLength=1 URL string `json:"url"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 61e81fd..608d47e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1292,6 +1292,7 @@ func (in *TargetSourceSpec) DeepCopyInto(out *TargetSourceSpec) { *out = new(ProviderSpec) (*in).DeepCopyInto(*out) } + in.Webhook.DeepCopyInto(&out.Webhook) if in.TargetLabels != nil { in, out := &in.TargetLabels, &out.TargetLabels *out = make(map[string]string, len(*in)) @@ -1477,3 +1478,23 @@ func (in *TunnelTargetPolicyStatus) DeepCopy() *TunnelTargetPolicyStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WebhookSpec) DeepCopyInto(out *WebhookSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookSpec. +func (in *WebhookSpec) DeepCopy() *WebhookSpec { + if in == nil { + return nil + } + out := new(WebhookSpec) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index f373822..b385d8e 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -67,6 +67,11 @@ spec: targetProfile: minLength: 1 type: string + webhook: + properties: + enabled: + type: boolean + type: object required: - provider - targetProfile diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go index c716965..128305a 100644 --- a/internal/controller/discovery/supervisor.go +++ b/internal/controller/discovery/supervisor.go @@ -14,19 +14,20 @@ type RestartPolicy struct { } type Component struct { - Name string - Run func(ctx context.Context) error - Policy RestartPolicy + Name string + Run func(ctx context.Context) error + Policy RestartPolicy + DegradeOnFailure bool } type Supervisor struct { ctx context.Context cancel context.CancelFunc - stopped bool - mu sync.Mutex + wg sync.WaitGroup - components []Component + mu sync.Mutex + stopped bool } func NewSupervisor(parent context.Context) *Supervisor { @@ -37,53 +38,6 @@ func NewSupervisor(parent context.Context) *Supervisor { } } -func (s *Supervisor) AddComponent(c Component) { - s.components = append(s.components, c) -} - -func (s *Supervisor) runComponent(c Component) { - logger := log.FromContext(s.ctx).WithValues( - "component", c.Name, - ) - - failures := 0 - - for { - err := c.Run(s.ctx) - if s.ctx.Err() != nil { - return - } - - failures++ - logger.Error(err, - "Component failed", - "attempt", failures, - ) - - if failures >= c.Policy.MaxRestarts { - logger.Error(err, - "Component exceeded restart limit; stopping discovery pipeline", - "restarts", failures, - ) - s.Stop() - return - } - - select { - case <-time.After(c.Policy.Backoff): - case <-s.ctx.Done(): - return - } - } -} - -func (s *Supervisor) Run() { - for _, c := range s.components { - component := c - go s.runComponent(component) - } -} - func (s *Supervisor) Stop() { s.mu.Lock() defer s.mu.Unlock() @@ -98,3 +52,55 @@ func (s *Supervisor) Stop() { func (s *Supervisor) Done() <-chan struct{} { return s.ctx.Done() } + +func (s *Supervisor) Wait() { + s.wg.Wait() +} + +func (s *Supervisor) RunComponent(component Component) { + s.wg.Add(1) + + go func() { + defer s.wg.Done() + + logger := log.FromContext(s.ctx).WithValues("component", component.Name) + failures := 0 + + for { + logger.Info("starting component") + err := component.Run(s.ctx) + + if s.ctx.Err() != nil { + logger.Info("component stopped due to pipeline shutdown") + return + } + + failures++ + logger.Error(err, + "component failed to run", + "attempt", failures, + "max", component.Policy.MaxRestarts, + ) + + if failures >= component.Policy.MaxRestarts { + if component.DegradeOnFailure { + logger.Error(err, + "component permanently failed; shutting down pipeline", + ) + s.Stop() + } else { + logger.Info( + "optional component permanently failed; continuing without it", + ) + } + return + } + + select { + case <-time.After(component.Policy.Backoff): + case <-s.ctx.Done(): + return + } + } + }() +} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 68f47eb..a687e80 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -167,16 +167,6 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName return err } - // Create loader instance - loader, err := discovery.NewLoader( - key, - targetSource.Spec, - core.LoaderConfig{ChunkSize: r.ChunkSize}, - ) - if err != nil { - return err - } - // Create target applier instance applier := discovery.NewTargetApplier( r.Client, @@ -184,34 +174,59 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName targetSource, targetChannel, ) - - supervisor.AddComponent(discovery.Component{ - Name: "loader", - Run: func(ctx context.Context) error { - return loader.Start(ctx, key, targetSource.Spec, targetChannel) - }, - Policy: discovery.RestartPolicy{ - MaxRestarts: pipelineMaxRestarts, - Backoff: pipelineBackoff, - }, - }) - - supervisor.AddComponent(discovery.Component{ + // Start target applier + applierReady := make(chan struct{}) + supervisor.RunComponent(discovery.Component{ Name: "target-applier", - Run: applier.Run, Policy: discovery.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, + DegradeOnFailure: true, + Run: func(ctx context.Context) error { + close(applierReady) + return applier.Run(ctx) + }, }) + // Wait for applier to be ready before starting loader + select { + case <-applierReady: + case <-supervisor.Done(): + return nil + } - supervisor.Run() + // Create loader instance + loaderConfigured := targetSource.Spec.Provider != nil + webhookConfigured := targetSource.Spec.Webhook.Enabled != nil + if loaderConfigured { + loader, err := discovery.NewLoader( + key, + targetSource.Spec, + core.LoaderConfig{ChunkSize: r.ChunkSize}, + ) + if err != nil { + supervisor.Stop() + return err + } + + supervisor.RunComponent(discovery.Component{ + Name: "loader", + Policy: discovery.RestartPolicy{ + MaxRestarts: pipelineMaxRestarts, + Backoff: pipelineBackoff, + }, + DegradeOnFailure: !webhookConfigured, + Run: func(ctx context.Context) error { + return loader.Start(ctx, key, targetSource.Spec, targetChannel) + }, + }) + } go func() { <-supervisor.Done() + supervisor.Wait() // Wait for components to exit logger.Info("Pipeline stopped; performing final cleanup") - close(targetChannel) r.DiscoveryRegistry.Unregister(key) r.stopDiscovery(key) From b8a6d272d479a97f05b5adeb6f9081520a236f8e Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Sat, 25 Apr 2026 09:31:29 +0000 Subject: [PATCH 045/110] refactor code --- internal/controller/discovery/supervisor.go | 54 +++++----- .../controller/targetsource_controller.go | 99 ++++++++++--------- 2 files changed, 87 insertions(+), 66 deletions(-) diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go index 128305a..710381e 100644 --- a/internal/controller/discovery/supervisor.go +++ b/internal/controller/discovery/supervisor.go @@ -8,18 +8,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) -type RestartPolicy struct { - MaxRestarts int - Backoff time.Duration -} - -type Component struct { - Name string - Run func(ctx context.Context) error - Policy RestartPolicy - DegradeOnFailure bool -} - +// Supervisor coordinates the runtime lifecycle of pipeline components +// +// Guarantees: +// - Each component is restarted independently +// - Permanent failure escalates according to policy +// - Stop() cancels all components +// - Wait() blocks until all goroutines exit type Supervisor struct { ctx context.Context cancel context.CancelFunc @@ -30,14 +25,30 @@ type Supervisor struct { stopped bool } -func NewSupervisor(parent context.Context) *Supervisor { - ctx, cancel := context.WithCancel(parent) +// RestartPolicy defines the restart behavior for a component +type RestartPolicy struct { + MaxRestarts int + Backoff time.Duration +} + +type ComponentSpec struct { + Name string + Run func(ctx context.Context) error + Policy RestartPolicy + // EscalatesOnFailure indicates whether a permanent failure of this component should shut down the entire pipeline + EscalatesOnFailure bool +} + +// NewSupervisor creates a new Supervisor with a cancellable context +func NewSupervisor(parentCtx context.Context) *Supervisor { + ctx, cancel := context.WithCancel(parentCtx) return &Supervisor{ ctx: ctx, cancel: cancel, } } +// Stop signals all supervised components to stop by canceling the context func (s *Supervisor) Stop() { s.mu.Lock() defer s.mu.Unlock() @@ -49,15 +60,14 @@ func (s *Supervisor) Stop() { s.cancel() } -func (s *Supervisor) Done() <-chan struct{} { - return s.ctx.Done() -} +// Done returns a channel that is closed when the pipeline is stopped +func (s *Supervisor) Done() <-chan struct{} { return s.ctx.Done() } -func (s *Supervisor) Wait() { - s.wg.Wait() -} +// Wait blocks until all supervised components have exited +func (s *Supervisor) Wait() { s.wg.Wait() } -func (s *Supervisor) RunComponent(component Component) { +// StartSupervisedComponent starts and supervises a component +func (s *Supervisor) StartSupervisedComponent(component ComponentSpec) { s.wg.Add(1) go func() { @@ -83,7 +93,7 @@ func (s *Supervisor) RunComponent(component Component) { ) if failures >= component.Policy.MaxRestarts { - if component.DegradeOnFailure { + if component.EscalatesOnFailure { logger.Error(err, "component permanently failed; shutting down pipeline", ) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index a687e80..f04eced 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -43,17 +43,26 @@ const ( pipelineBackoff = 3 * time.Second ) -type runningSource struct { +// pipelineHandle represents a controller-owned handle to a running pipeline +// The controller never manipulates internals; it only invokes cancel() +type pipelineHandle struct { cancel context.CancelFunc } // TargetSourceReconciler reconciles a TargetSource object +// +// Responsibilities: +// - Ensure at most one pipeline per TargetSource +// - Start pipelines on reconcile +// - Stop pipelines on deletion or NotFound +// - Delegate runtime failure handling to the Supervisor type TargetSourceReconciler struct { client.Client Scheme *runtime.Scheme - mu sync.Mutex - running map[types.NamespacedName]runningSource + mu sync.Mutex + // runningPipelines tracks currently active pipelines by NamespacedName + runningPipelines map[types.NamespacedName]pipelineHandle BufferSize int ChunkSize int @@ -69,47 +78,43 @@ type TargetSourceReconciler struct { // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx).WithName("targetsource controller").WithValues( - "targetsource", req.NamespacedName, - ) + logger := log.FromContext(ctx). + WithName("targetsource controller"). + WithValues("targetsource", req.NamespacedName) - targetSource, err := r.getTargetSource(ctx, req.NamespacedName) + targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName) if err != nil { // If the TargetSource no longer exists, ensure runtime cleanup if client.IgnoreNotFound(err) == nil { - logger.Info("TargetSource not found, ensuring cleanup") - r.stopDiscovery(req.NamespacedName) + logger.Info("TargetSource not found; stopping discovery pipeline") + r.stopDiscoveryPipeline(req.NamespacedName) return ctrl.Result{}, nil } return ctrl.Result{}, err } - // Handle deletion with finalizer if !targetSource.DeletionTimestamp.IsZero() { - return r.handleTargetSourceDeletion(ctx, req.NamespacedName, targetSource) + return r.reconcileDeletion(ctx, req.NamespacedName, targetSource) } - // Ensure finalizer is set if err := r.ensureFinalizer(ctx, targetSource); err != nil { return ctrl.Result{}, err } - // Check if pipeline is already running - if r.isPipelineRunning(req.NamespacedName) { + if r.hasPipelineRunning(req.NamespacedName) { return ctrl.Result{}, nil } - // Start discovery pipeline if err := r.startDiscoveryPipeline(req.NamespacedName, targetSource, logger); err != nil { return ctrl.Result{}, err } - logger.Info("TargetSource pipeline started") + logger.Info("Discover pipeline started") return ctrl.Result{}, nil } -// getTargetSource retrieves a TargetSource by name, handling cleanup if not found -func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key types.NamespacedName) (*gnmicv1alpha1.TargetSource, error) { +// fetchTargetSource retrieves a TargetSource by name, handling cleanup if not found +func (r *TargetSourceReconciler) fetchTargetSource(ctx context.Context, key types.NamespacedName) (*gnmicv1alpha1.TargetSource, error) { var targetSource gnmicv1alpha1.TargetSource if err := r.Get(ctx, key, &targetSource); err != nil { return nil, err @@ -117,12 +122,20 @@ func (r *TargetSourceReconciler) getTargetSource(ctx context.Context, key types. return &targetSource, nil } -// handleTargetSourceDeletion stops the discovery pipeline and removes the finalizer -func (r *TargetSourceReconciler) handleTargetSourceDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { +// hasPipelineRunning checks if a discovery pipeline is already running for the given key +func (r *TargetSourceReconciler) hasPipelineRunning(key types.NamespacedName) bool { + r.mu.Lock() + defer r.mu.Unlock() + _, exists := r.runningPipelines[key] + return exists +} + +// reconcileDeletion stops the discovery pipeline and removes the finalizer +func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { logger := log.FromContext(ctx) logger.Info("TargetSource is being deleted, stopping pipeline", "name", key) - r.stopDiscovery(key) + r.stopDiscoveryPipeline(key) // Remove finalizer if exists if controllerutil.ContainsFinalizer(targetSource, targetSourceFinalizer) { @@ -149,16 +162,13 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour return nil } -// isPipelineRunning checks if a discovery pipeline is already running for the given key -func (r *TargetSourceReconciler) isPipelineRunning(key types.NamespacedName) bool { - r.mu.Lock() - defer r.mu.Unlock() - - _, exists := r.running[key] - return exists -} - -// startDiscoveryPipeline creates and starts the loader and target manager +// startDiscoveryPipeline creates and starts a discover pipeline for a TargetSource +// +// Pipeline semantics: +// 1. target-applier is mandatory and must start first +// 2. loader is optional and conditional on spec +// 3. Permanent failure of required components shuts down the pipeline +// 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error { supervisor := discovery.NewSupervisor(context.Background()) @@ -176,15 +186,15 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName ) // Start target applier applierReady := make(chan struct{}) - supervisor.RunComponent(discovery.Component{ + supervisor.StartSupervisedComponent(discovery.ComponentSpec{ Name: "target-applier", Policy: discovery.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, - DegradeOnFailure: true, + EscalatesOnFailure: true, Run: func(ctx context.Context) error { - close(applierReady) + close(applierReady) // Signals that applier started successfully return applier.Run(ctx) }, }) @@ -209,31 +219,32 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName return err } - supervisor.RunComponent(discovery.Component{ + supervisor.StartSupervisedComponent(discovery.ComponentSpec{ Name: "loader", Policy: discovery.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, - DegradeOnFailure: !webhookConfigured, + EscalatesOnFailure: !webhookConfigured, Run: func(ctx context.Context) error { return loader.Start(ctx, key, targetSource.Spec, targetChannel) }, }) } + // Monitor supervisor in a separate goroutine to handle shutdown and cleanup go func() { <-supervisor.Done() supervisor.Wait() // Wait for components to exit - logger.Info("Pipeline stopped; performing final cleanup") + logger.Info("Pipeline stopped; cleaning up") close(targetChannel) r.DiscoveryRegistry.Unregister(key) - r.stopDiscovery(key) + r.stopDiscoveryPipeline(key) }() r.mu.Lock() - r.running[key] = runningSource{ + r.runningPipelines[key] = pipelineHandle{ cancel: func() { supervisor.Stop() }, @@ -243,12 +254,12 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName return nil } -// stopDiscovery stops and removes a running discovery pipeline -func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) { +// stopDiscoveryPipeline stops and removes a running discovery pipeline +func (r *TargetSourceReconciler) stopDiscoveryPipeline(key types.NamespacedName) { r.mu.Lock() - running, ok := r.running[key] + running, ok := r.runningPipelines[key] if ok { - delete(r.running, key) + delete(r.runningPipelines, key) } r.mu.Unlock() @@ -259,7 +270,7 @@ func (r *TargetSourceReconciler) stopDiscovery(key types.NamespacedName) { // SetupWithManager sets up the controller with the Manager. func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { - r.running = make(map[types.NamespacedName]runningSource) + r.runningPipelines = make(map[types.NamespacedName]pipelineHandle) return ctrl.NewControllerManagedBy(mgr). For(&gnmicv1alpha1.TargetSource{}). From eedfedf930d6f78ef9ca430115bb121ee9db129c Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Sat, 25 Apr 2026 12:55:52 +0000 Subject: [PATCH 046/110] improve context handling of and target applier semantics --- internal/controller/discovery/core/helpers.go | 14 +- internal/controller/discovery/core/types.go | 5 +- .../controller/discovery/target_applier.go | 209 ++++++++++++++---- 3 files changed, 184 insertions(+), 44 deletions(-) diff --git a/internal/controller/discovery/core/helpers.go b/internal/controller/discovery/core/helpers.go index 843f30e..f24b50c 100644 --- a/internal/controller/discovery/core/helpers.go +++ b/internal/controller/discovery/core/helpers.go @@ -2,6 +2,7 @@ package core import ( "context" + "fmt" ) // sendMessages sends discovery messages over a channel in a context-aware manner @@ -32,13 +33,15 @@ func forEachChunk(total, chunkSize int, fn func(start, end int) error) error { func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chunkSize int) []DiscoverySnapshot { var snapshots []DiscoverySnapshot totalTargets := len(targets) + totalChunks := (totalTargets + chunkSize - 1) / chunkSize _ = forEachChunk(totalTargets, chunkSize, func(i, end int) error { chunk := targets[i:end] snapshots = append(snapshots, DiscoverySnapshot{ Targets: chunk, SnapshotID: snapshotID, - IsLastChunk: (end == totalTargets), + ChunkIndex: i / chunkSize, + TotalChunks: totalChunks, }) return nil }) @@ -48,8 +51,11 @@ func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chu // SendSnapshot sends discovered targets as a snapshot over a channel in chunks func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets []DiscoveredTarget, snapshotID string, chunkSize int) error { - snapshots := createDiscoverySnapshots(targets, snapshotID, chunkSize) + if len(targets) == 0 { + return fmt.Errorf("no targets in Snapshot") + } + snapshots := createDiscoverySnapshots(targets, snapshotID, chunkSize) for _, snapshot := range snapshots { // Convert DiscoverySnapshot to DiscoveryMessage messages := make([]DiscoveryMessage, 1) @@ -73,6 +79,10 @@ func eventsToMessages(events []DiscoveryEvent) []DiscoveryMessage { // SendEvents sends discovery messages over channel in a context-aware manner func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent, chunkSize int) error { + if len(events) == 0 { + return fmt.Errorf("no events to process") + } + messages := eventsToMessages(events) total := len(messages) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 61209fd..3f6957a 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -26,7 +26,8 @@ type DiscoveryEvent struct { } type DiscoverySnapshot struct { - Targets []DiscoveredTarget SnapshotID string - IsLastChunk bool + ChunkIndex int + TotalChunks int + Targets []DiscoveredTarget } diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_applier.go index c60f2b8..ee127c5 100644 --- a/internal/controller/discovery/target_applier.go +++ b/internal/controller/discovery/target_applier.go @@ -2,6 +2,7 @@ package discovery import ( "context" + "fmt" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -12,13 +13,23 @@ import ( "github.com/go-logr/logr" ) +type snapshotBuffer struct { + snapshotID string + totalChunks int + received map[int][]core.DiscoveredTarget + complete bool +} + // TargetApplier consumes discovered targets and applies them to Kubernetes type TargetApplier struct { - client client.Client - scheme *runtime.Scheme - targetSource *gnmicv1alpha1.TargetSource - in <-chan []core.DiscoveryMessage - collected map[string][]core.DiscoveredTarget + client client.Client + scheme *runtime.Scheme + targetSource *gnmicv1alpha1.TargetSource + in <-chan []core.DiscoveryMessage + queue []core.DiscoveryMessage + activeSnapshot *snapshotBuffer + // Events are deferred while snapshot is in progress + defferedEvents []core.DiscoveryEvent } // NewTargetApplier wires a TargetApplier instance @@ -28,47 +39,43 @@ func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ scheme: s, targetSource: ts, in: in, - collected: make(map[string][]core.DiscoveredTarget), } } // Run is a long‑running loop that receives target snapshots // and reconciles Target CRs accordingly -func (m *TargetApplier) Run(ctx context.Context) error { +func (a *TargetApplier) Run(ctx context.Context) error { logger := log.FromContext(ctx). WithValues( - "name", m.targetSource.Name, - "namespace", m.targetSource.Namespace, + "name", a.targetSource.Name, + "namespace", a.targetSource.Namespace, ) - logger.Info("target applier started") - queue := []core.DiscoveryMessage{} - for ctx.Err() == nil { select { - case batch, ok := <-m.in: + case batch, ok := <-a.in: if !ok { // Channel closed, pipeline is shutting down logger.Info("input channel closed, stopping target applier") return nil } - queue = append(queue, batch...) + a.queue = append(a.queue, batch...) case <-ctx.Done(): logger.Info("context canceled, stopping target applier") return nil } - for len(queue) > 0 { + for len(a.queue) > 0 { if ctx.Err() != nil { - return ctx.Err() + return nil // why return nil? } - msg := queue[0] - queue = queue[1:] + msg := a.queue[0] + a.queue = a.queue[1:] - if err := m.handleMessage(ctx, msg, logger); err != nil { + if err := a.processMessage(ctx, msg, logger); err != nil { // Returning error lets the supervisor (controller) // tear down and restart the pipeline via reconciliation // Q: when to return an error vs just log and continue? @@ -82,7 +89,7 @@ func (m *TargetApplier) Run(ctx context.Context) error { return nil } -func (m *TargetApplier) handleMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { +func (a *TargetApplier) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { if err := ctx.Err(); err != nil { return err } @@ -94,12 +101,10 @@ func (m *TargetApplier) handleMessage(ctx context.Context, message core.Discover logger.Info( "received snapshot chunk", "snapshotID", msg.SnapshotID, + "index", msg.ChunkIndex, "targetCount", len(msg.Targets), ) - m.collected[msg.SnapshotID] = append(m.collected[msg.SnapshotID], msg.Targets...) - if msg.IsLastChunk { - m.processSnapshot(msg.SnapshotID, logger) - } + return a.processSnapshot(ctx, msg, logger) case core.DiscoveryEvent: // Process individual event-driven update @@ -107,31 +112,155 @@ func (m *TargetApplier) handleMessage(ctx context.Context, message core.Discover "received discovery event", "target", msg.Target.Name, ) - switch msg.Event { - case core.CREATE: - logger.Info("Would create target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) - case core.UPDATE: - logger.Info("Would update target", "name", msg.Target.Name, "address", msg.Target.Address, "labels", msg.Target.Labels) - case core.DELETE: - logger.Info("Would delete target", "name", msg.Target.Name) + return a.processEvent(ctx, msg, logger) + + default: + return fmt.Errorf("unknonw discovery message type %T", msg) + } +} + +// processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly +func (a *TargetApplier) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { + if a.activeSnapshot == nil { + a.startNewSnapshot(chunk, logger) + return nil + } + + snapshot := a.activeSnapshot + // Check if a new snapshot arrived + if snapshot.snapshotID != chunk.SnapshotID { + // If current snapshot is complete apply it first + if snapshot.complete { + if err := a.applySnapshot(ctx, snapshot, logger); err != nil { + return err + } + } else { + // If a new snapshot is started before the old one completed + // the old one can be discarded + logger.Info( + "discarding incomplete snapshot", + "snapshotID", snapshot.snapshotID, + ) } + + // Start collecting the new snapshot + a.startNewSnapshot(chunk, logger) + return nil + } + + return a.collectSnapshot(chunk, logger) +} + +func (a *TargetApplier) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { + a.activeSnapshot = &snapshotBuffer{ + snapshotID: chunk.SnapshotID, + totalChunks: chunk.TotalChunks, + received: make(map[int][]core.DiscoveredTarget), + complete: false, + } + // Delete buffered events that will be current with new snapshot + a.defferedEvents = nil + + a.collectSnapshot(chunk, logger) +} + +func (a *TargetApplier) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { + snapshot := a.activeSnapshot + + if chunk.TotalChunks != snapshot.totalChunks { + logger.Error(nil, "snapshot totalChunks mismatch", "snapshotID", snapshot.snapshotID) + } + if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks { + logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex) + a.activeSnapshot = nil + return nil + } + if _, exists := snapshot.received[chunk.ChunkIndex]; exists { + logger.Error(nil, "duplicate snapshot chunk", "index", chunk.ChunkIndex) + a.activeSnapshot = nil + return nil + } + + snapshot.received[chunk.ChunkIndex] = chunk.Targets + + if len(snapshot.received) == snapshot.totalChunks { + snapshot.complete = true } return nil } -// processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly -func (m *TargetApplier) processSnapshot(snapshotID string, logger logr.Logger) { - targets := m.collected[snapshotID] - delete(m.collected, snapshotID) +func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { + select { + case <-ctx.Done(): + a.activeSnapshot = nil + return nil + default: + } - logger.Info("Processing full snapshot", "snapshotID", snapshotID, "totalTargets", len(targets)) + var allTargets []core.DiscoveredTarget + for i := 0; i < snapshot.totalChunks; i++ { + select { + case <-ctx.Done(): + a.activeSnapshot = nil + return nil + default: + } - if m.targetSource.Spec.Provider.HTTP != nil { - logger.Info("Would delete all existing targets for targetsource", "targetsource", m.targetSource.Name) + chunk, ok := snapshot.received[i] + if !ok { + logger.Error(nil, "missing snapshot chunk", "index", i) + a.activeSnapshot = nil + return nil + } + allTargets = append(allTargets, chunk...) } - for _, target := range targets { - logger.Info("Would create target", "name", target.Name, "address", target.Address, "labels", target.Labels) + logger.Info( + "applying snapshot", + "snapshotID", snapshot.snapshotID, + "targetCount", len(allTargets), + ) + + // apply all targets + // a.applyTargets + + // Replay deffered events + for _, event := range a.defferedEvents { + select { + case <-ctx.Done(): + return nil + default: + } + if err := a.applyEvent(ctx, event, logger); err != nil { + return err + } } + + a.activeSnapshot = nil + a.defferedEvents = nil + return nil +} + +func (a *TargetApplier) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { + // If snapshot collecting is active defer events + if a.activeSnapshot != nil { + a.defferedEvents = append(a.defferedEvents, event) + return nil + } + + // Apply events + return a.applyEvent(ctx, event, logger) +} + +func (a *TargetApplier) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { + switch event.Event { + case core.CREATE: + logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) + case core.UPDATE: + logger.Info("Would update target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) + case core.DELETE: + logger.Info("Would delete target", "name", event.Target.Name) + } + return nil } From ff4f2bbfe7ffe9f7c3a4715268e692964b9e1ad8 Mon Sep 17 00:00:00 2001 From: Janooski Date: Sun, 26 Apr 2026 09:44:15 +0000 Subject: [PATCH 047/110] fix import --- internal/apiserver/apiserver.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 7f125cf..0e65872 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -9,7 +9,6 @@ import ( "net/http" "github.com/gin-gonic/gin" - "github.com/gin-gonic/gin/binding" "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/registry" From ffbfd47953a6940057f76f7eaf8e6ebfde241807 Mon Sep 17 00:00:00 2001 From: Janooski Date: Sun, 26 Apr 2026 14:29:34 +0000 Subject: [PATCH 048/110] update api contract --- cmd/main.go | 2 +- .../operator.gnmic.dev_targetsources.yaml | 4 ++-- internal/apiserver/apiserver.go | 2 +- internal/apiserver/openapi.yaml | 19 +++++++++---------- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 6ab354f..5db5f6e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -238,7 +238,7 @@ func main() { setupLog.Error(err, "unable to intialize gin API server") os.Exit(1) } - api.ChunkSize = discoveryChunkSize + // api.ChunkSize = discoveryChunkSize api.DiscoveryRegistry = discoveryRegistry err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error) diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 95f5e15..5c96244 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -57,8 +57,8 @@ spec: type: object pull: properties: - url: - type: string + enabled: + type: boolean type: object type: object x-kubernetes-validations: diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index ed92e1b..ccc8f15 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -1,6 +1,6 @@ package apiserver -//go:generate go tool oapi-codegen -config cfg.yaml openapi.yaml +// go:generate go tool oapi-codegen -config cfg.yaml openapi.yaml // or use go generate ./internal/apiserver in the console (install from https://github.com/oapi-codegen/oapi-codegen) import ( diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index bd66bb8..d84f265 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -21,23 +21,22 @@ paths: application/json: schema: type: object - required: [TargetSource, Target] properties: - TargetSource: - $ref: '#/components/schemas/TargetSource' - Target: - type: array - items: - $ref: '#/components/schemas/Target' + Targets: + $ref: '#/components/schemas/Targets' components: schemas: - TargetSource: + Targets: type: object properties: - namespace: + targetSourceNameSpace: type: string - name: + TargetSourceName: type: string + TargetList: + type: array + items: + $ref: '#/components/schemas/Target' Target: type: object properties: From 8d7ee19f7c9a2f7fa2c01bc32bbd24a7f46a17e8 Mon Sep 17 00:00:00 2001 From: Janooski Date: Sun, 26 Apr 2026 15:43:22 +0000 Subject: [PATCH 049/110] CreateTargets works for complete POST request --- internal/apiserver/apiserver.go | 61 +++++++++++++++------------------ internal/apiserver/gen.go | 24 ++++++------- internal/apiserver/temp.md | 6 ++++ 3 files changed, 45 insertions(+), 46 deletions(-) create mode 100644 internal/apiserver/temp.md diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index ccc8f15..b24c7d7 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -1,7 +1,8 @@ package apiserver -// go:generate go tool oapi-codegen -config cfg.yaml openapi.yaml -// or use go generate ./internal/apiserver in the console (install from https://github.com/oapi-codegen/oapi-codegen) +//go:generate go tool oapi-codegen -config cfg.yaml openapi.yaml +// To generate code, install openapi-codegen from https://github.com/oapi-codegen/oapi-codegen) +// Then use: go generate ./internal/apiserver import ( "context" @@ -50,49 +51,41 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { c.JSON(200, plan) } -// CreateTargets binds payload to Target struct defined in openapi.yaml and sends it to pull loader +// CreateTargets binds payload to payloadTargets struct defined in openapi contract. Passes func (a *APIServer) CreateTargets(c *gin.Context) { - // logger.Info("Create Targets called") - var payloadTarget []Target - var payloadTargetSource TargetSource + var payloadTargets Targets fmt.Println("Binding Target to PayloadTarget") - // https://gin-gonic.com/en/docs/binding/bind-body-into-different-structs/ - if err := c.ShouldBindBodyWithJSON(&payloadTarget); err != nil { + if err := c.ShouldBind(&payloadTargets); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) fmt.Printf("err: %s", err.Error) return } - fmt.Printf("payloadTarget: %s", payloadTarget) - if err := c.ShouldBindBodyWithJSON(&payloadTargetSource); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - - // error {"error":"json: cannot unmarshal object into Go value of type []apiserver.Target"} targets := []core.DiscoveryEvent{} - for _, target := range payloadTarget { - event := core.CREATE - switch *target.Operation { - case Create: - event = core.CREATE - case Delete: - event = core.DELETE + if len(*payloadTargets.TargetList) > 0 { // doesn't work on empty TargetList + for _, target := range *payloadTargets.TargetList { + event := core.CREATE + switch *target.Operation { + case Create: + event = core.CREATE + case Delete: + event = core.DELETE + } + targets = append(targets, core.DiscoveryEvent{ + Target: core.DiscoveredTarget{ + Name: *target.Name, + Address: *target.Address, + Labels: map[string]string{"key": "Is this a tag?"}, + }, + Event: event, + }) } - targets = append(targets, core.DiscoveryEvent{ - Target: core.DiscoveredTarget{ - Name: *target.Name, - Address: *target.Address, - Labels: map[string]string{"key": "Is this a tag?"}, - }, - Event: event, - }) } key := types.NamespacedName{ - Namespace: *payloadTargetSource.Namespace, - Name: *payloadTargetSource.Name, + Namespace: *payloadTargets.TargetSourceNameSpace, + Name: *payloadTargets.TargetSourceName, } ch, ok := a.DiscoveryRegistry.Get(key) if !ok { @@ -100,7 +93,7 @@ func (a *APIServer) CreateTargets(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": "Target Source doesn't exist"}) return } - + fmt.Println("Preparing SendEvents") core.SendEvents(context.Background(), ch, targets, 10) // make number constant - c.JSON(http.StatusOK, payloadTarget) + c.JSON(http.StatusOK, payloadTargets) } diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index acc383b..84321bd 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -46,10 +46,11 @@ type Target struct { // TargetOperation defines model for Target.Operation. type TargetOperation string -// TargetSource defines model for TargetSource. -type TargetSource struct { - Name *string `json:"name,omitempty"` - Namespace *string `json:"namespace,omitempty"` +// Targets defines model for Targets. +type Targets struct { + TargetList *[]Target `json:"TargetList,omitempty"` + TargetSourceName *string `json:"TargetSourceName,omitempty"` + TargetSourceNameSpace *string `json:"targetSourceNameSpace,omitempty"` } // ServerInterface represents all server handlers. @@ -131,14 +132,13 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/4RSTW/bMAz9KwK3oxG73c23oRiKHLYVa2/DDprMJCpsSSOpAUHg/z7oY0k8Z+hJBPlI", - "ke+9Exg/Be/QCUN/AjYHnHQOXzTtUVIUyAcksZjzehgIOYdyDAg9sJB1e5gbcHrCm4U0QIv1LlXRxQn6", - "72AItSA0MOCIgvCjWTcG8js73h4qep/XsILT7X1qQhPpI8yXhP/5ikYSolz57CMZXN/633tSgYM2t6rr", - "b1LKup3PYCvpHNh/+bw16mvmxZP69un5RX182kIDv5E4MwXdptvcVfqcDhZ6+LC533TQQNByyCu2hcVy", - "R84Ez1m2M+fbAXp4WMAaIOTgHZc777u79BjvBF1u1iGM1uT29pWLbsUca5YuTjkr8Z5wBz28ay/2aqu3", - "2gpfybMW4+0pFZsIJvwVLeGQnLUo/p175a9rYQZkQzYUc1Yoq8LqoDgag8y7OI7FQRynSdPxzKiS2mGd", - "kgOqpa65pQ2jzgRWlpbCPKI8jJEF6SnBVsp06VkueYVXhBLJ4fDPco8oyhSYyt/P8zz/CQAA//+Shfmd", - "7gMAAA==", + "H4sIAAAAAAAC/4yTz2vdMAzH/xWj7RiStLvlNsooD7au7PU2dvAcJc8lsY2kDB4l//uwnf5Ikwc9xchf", + "Wfp+pDyB8WPwDp0wNE/A5oSjTscHTT1KPAXyAUksprhuW0JORzkHhAZYyLoe5gKcHnH3Ij6gxXoXb9FN", + "IzS/wRBqQSigxQEF4U+xTQzkOzvsPyq6T21YwXG/nyWgifQZ5teA//uIRqIiu+StzXzx3bKsSnwm7KCB", + "T9Urt2qBVi3ENnWfqxz9RAbvLjGSd6Jj0GZPubURQ9Z1PomtRFzQ3/04GPUzcfekfn07Pqiv9wco4B8S", + "p0lAXdbl1TIep4OFBr6U12UNBQQtp2S4ylN6y8lnJi8zPbTQwM1KVgAhB+84w7yur+LHeCfoUrIOYbAm", + "pVePnPcic7w0ig/i50uAWmRDNuQlfH5TZXet4skYZO6mYcibwtM4ajq/OFOyZFin5IRqzTelVGHQycjy", + "36wB3aLcDBML0n2UbQjV8bNu8o1eEcpEDtt3zd2iKJNlKpWf53n+HwAA////AVrz1gMAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md new file mode 100644 index 0000000..90f3fb0 --- /dev/null +++ b/internal/apiserver/temp.md @@ -0,0 +1,6 @@ +## CURL request +curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' + + +## Empty TargetList +curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' \ No newline at end of file From a66accbbcac43a0cdbefa4f59231ca57fca1635f Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 26 Apr 2026 18:23:19 -0600 Subject: [PATCH 050/110] moved finalizer label into const file --- internal/controller/const.go | 2 ++ internal/controller/targetsource_controller.go | 10 ++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/controller/const.go b/internal/controller/const.go index b5196b8..5ef2e8f 100644 --- a/internal/controller/const.go +++ b/internal/controller/const.go @@ -21,6 +21,8 @@ const ( LabelCertType = "operator.gnmic.dev/cert-type" LabelValueCertTypeClient = "client" LabelValueCertTypeTunnel = "tunnel" + + LabelTargetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer" ) const ( diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index f04eced..232c624 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -37,8 +37,6 @@ import ( ) const ( - targetSourceFinalizer = "operator.gnmic.dev/targetsource-finalizer" - pipelineMaxRestarts = 5 pipelineBackoff = 3 * time.Second ) @@ -138,8 +136,8 @@ func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key type r.stopDiscoveryPipeline(key) // Remove finalizer if exists - if controllerutil.ContainsFinalizer(targetSource, targetSourceFinalizer) { - controllerutil.RemoveFinalizer(targetSource, targetSourceFinalizer) + if controllerutil.ContainsFinalizer(targetSource, LabelTargetSourceFinalizer) { + controllerutil.RemoveFinalizer(targetSource, LabelTargetSourceFinalizer) if err := r.Update(ctx, targetSource); err != nil { return ctrl.Result{}, err } @@ -150,11 +148,11 @@ func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key type // ensureFinalizer adds the finalizer if not present and updates the TargetSource func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSource *gnmicv1alpha1.TargetSource) error { - if controllerutil.ContainsFinalizer(targetSource, targetSourceFinalizer) { + if controllerutil.ContainsFinalizer(targetSource, LabelTargetSourceFinalizer) { return nil } - controllerutil.AddFinalizer(targetSource, targetSourceFinalizer) + controllerutil.AddFinalizer(targetSource, LabelTargetSourceFinalizer) if err := r.Update(ctx, targetSource); err != nil { return err } From 3b2d9258a06116738be182e567ee6f275c9ad0e4 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 26 Apr 2026 18:29:05 -0600 Subject: [PATCH 051/110] fixed typo --- internal/controller/discovery/target_applier.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_applier.go index ee127c5..3c714bd 100644 --- a/internal/controller/discovery/target_applier.go +++ b/internal/controller/discovery/target_applier.go @@ -29,7 +29,7 @@ type TargetApplier struct { queue []core.DiscoveryMessage activeSnapshot *snapshotBuffer // Events are deferred while snapshot is in progress - defferedEvents []core.DiscoveryEvent + deferredEvents []core.DiscoveryEvent } // NewTargetApplier wires a TargetApplier instance @@ -159,7 +159,7 @@ func (a *TargetApplier) startNewSnapshot(chunk core.DiscoverySnapshot, logger lo complete: false, } // Delete buffered events that will be current with new snapshot - a.defferedEvents = nil + a.deferredEvents = nil a.collectSnapshot(chunk, logger) } @@ -225,8 +225,8 @@ func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuf // apply all targets // a.applyTargets - // Replay deffered events - for _, event := range a.defferedEvents { + // Replay deferred events + for _, event := range a.deferredEvents { select { case <-ctx.Done(): return nil @@ -238,14 +238,14 @@ func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuf } a.activeSnapshot = nil - a.defferedEvents = nil + a.deferredEvents = nil return nil } func (a *TargetApplier) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { // If snapshot collecting is active defer events if a.activeSnapshot != nil { - a.defferedEvents = append(a.defferedEvents, event) + a.deferredEvents = append(a.deferredEvents, event) return nil } From 3ba86cb63c45a7f042a2051faca5f8ddfdc5b2ad Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 26 Apr 2026 19:21:01 -0600 Subject: [PATCH 052/110] restructured loaders package --- .../controller/discovery/loaders/http/{loader.go => http.go} | 0 .../discovery/loaders/http/{loader_test.go => http_test.go} | 0 .../controller/discovery/{loader.go => loaders/loaders.go} | 2 +- internal/controller/targetsource_controller.go | 3 ++- 4 files changed, 3 insertions(+), 2 deletions(-) rename internal/controller/discovery/loaders/http/{loader.go => http.go} (100%) rename internal/controller/discovery/loaders/http/{loader_test.go => http_test.go} (100%) rename internal/controller/discovery/{loader.go => loaders/loaders.go} (97%) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/http.go similarity index 100% rename from internal/controller/discovery/loaders/http/loader.go rename to internal/controller/discovery/loaders/http/http.go diff --git a/internal/controller/discovery/loaders/http/loader_test.go b/internal/controller/discovery/loaders/http/http_test.go similarity index 100% rename from internal/controller/discovery/loaders/http/loader_test.go rename to internal/controller/discovery/loaders/http/http_test.go diff --git a/internal/controller/discovery/loader.go b/internal/controller/discovery/loaders/loaders.go similarity index 97% rename from internal/controller/discovery/loader.go rename to internal/controller/discovery/loaders/loaders.go index 0d8ddd3..45bf9c1 100644 --- a/internal/controller/discovery/loader.go +++ b/internal/controller/discovery/loaders/loaders.go @@ -1,4 +1,4 @@ -package discovery +package loaders import ( "fmt" diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 232c624..77a3a35 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -31,6 +31,7 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/gnmic/operator/internal/controller/discovery/loaders" _ "github.com/gnmic/operator/internal/controller/discovery/loaders/all" "github.com/gnmic/operator/internal/controller/discovery/registry" "github.com/go-logr/logr" @@ -207,7 +208,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName loaderConfigured := targetSource.Spec.Provider != nil webhookConfigured := targetSource.Spec.Webhook.Enabled != nil if loaderConfigured { - loader, err := discovery.NewLoader( + loader, err := loaders.NewLoader( key, targetSource.Spec, core.LoaderConfig{ChunkSize: r.ChunkSize}, From d0ac86be2e389e91ef833bf5c278324af2df59bb Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 26 Apr 2026 19:21:13 -0600 Subject: [PATCH 053/110] restructured target handler --- internal/controller/discovery/client.go | 27 ---- .../{target_applier.go => target_handler.go} | 121 ++++++++++-------- .../controller/targetsource_controller.go | 20 +-- 3 files changed, 80 insertions(+), 88 deletions(-) delete mode 100644 internal/controller/discovery/client.go rename internal/controller/discovery/{target_applier.go => target_handler.go} (66%) diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go deleted file mode 100644 index 3bc7ef7..0000000 --- a/internal/controller/discovery/client.go +++ /dev/null @@ -1,27 +0,0 @@ -package discovery - -// File may become obsolete, depends on how the logic to compare desired vs. existing state will get implemented - -import ( - "context" - - "sigs.k8s.io/controller-runtime/pkg/client" - - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" -) - -func FetchExistingTargets(ctx context.Context, c client.Client, ts gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { - var targetList gnmicv1alpha1.TargetList - - err := c.List(ctx, &targetList, - client.InNamespace(ts.Namespace), - client.MatchingLabels{ - "gnmic.io/source": ts.Name, - }, - ) - if err != nil { - return nil, err - } - - return targetList.Items, nil -} diff --git a/internal/controller/discovery/target_applier.go b/internal/controller/discovery/target_handler.go similarity index 66% rename from internal/controller/discovery/target_applier.go rename to internal/controller/discovery/target_handler.go index 3c714bd..e8c0308 100644 --- a/internal/controller/discovery/target_applier.go +++ b/internal/controller/discovery/target_handler.go @@ -20,8 +20,9 @@ type snapshotBuffer struct { complete bool } -// TargetApplier consumes discovered targets and applies them to Kubernetes -type TargetApplier struct { +// TargetHandler consumes discovered targets and applies them to Kubernetes +type TargetHandler struct { + ctx context.Context client client.Client scheme *runtime.Scheme targetSource *gnmicv1alpha1.TargetSource @@ -32,9 +33,9 @@ type TargetApplier struct { deferredEvents []core.DiscoveryEvent } -// NewTargetApplier wires a TargetApplier instance -func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetApplier { - return &TargetApplier{ +// NewTargetHandler wires a TargetHandler instance +func NewTargetHandler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetHandler { + return &TargetHandler{ client: c, scheme: s, targetSource: ts, @@ -44,38 +45,40 @@ func NewTargetApplier(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ // Run is a long‑running loop that receives target snapshots // and reconciles Target CRs accordingly -func (a *TargetApplier) Run(ctx context.Context) error { - logger := log.FromContext(ctx). +func (c *TargetHandler) Run(ctx context.Context) error { + c.ctx = ctx + + logger := log.FromContext(c.ctx). WithValues( - "name", a.targetSource.Name, - "namespace", a.targetSource.Namespace, + "name", c.targetSource.Name, + "namespace", c.targetSource.Namespace, ) - logger.Info("target applier started") + logger.Info("target handler started") - for ctx.Err() == nil { + for c.ctx.Err() == nil { select { - case batch, ok := <-a.in: + case batch, ok := <-c.in: if !ok { // Channel closed, pipeline is shutting down - logger.Info("input channel closed, stopping target applier") + logger.Info("input channel closed, stopping target handler") return nil } - a.queue = append(a.queue, batch...) + c.queue = append(c.queue, batch...) case <-ctx.Done(): - logger.Info("context canceled, stopping target applier") + logger.Info("context canceled, stopping target handler") return nil } - for len(a.queue) > 0 { + for len(c.queue) > 0 { if ctx.Err() != nil { return nil // why return nil? } - msg := a.queue[0] - a.queue = a.queue[1:] + msg := c.queue[0] + c.queue = c.queue[1:] - if err := a.processMessage(ctx, msg, logger); err != nil { + if err := c.processMessage(c.ctx, msg, logger); err != nil { // Returning error lets the supervisor (controller) // tear down and restart the pipeline via reconciliation // Q: when to return an error vs just log and continue? @@ -85,11 +88,11 @@ func (a *TargetApplier) Run(ctx context.Context) error { } } - logger.Info("target applier stopped") + logger.Info("target handler stopped") return nil } -func (a *TargetApplier) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { +func (c *TargetHandler) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { if err := ctx.Err(); err != nil { return err } @@ -104,7 +107,7 @@ func (a *TargetApplier) processMessage(ctx context.Context, message core.Discove "index", msg.ChunkIndex, "targetCount", len(msg.Targets), ) - return a.processSnapshot(ctx, msg, logger) + return c.processSnapshot(ctx, msg, logger) case core.DiscoveryEvent: // Process individual event-driven update @@ -112,7 +115,7 @@ func (a *TargetApplier) processMessage(ctx context.Context, message core.Discove "received discovery event", "target", msg.Target.Name, ) - return a.processEvent(ctx, msg, logger) + return c.processEvent(ctx, msg, logger) default: return fmt.Errorf("unknonw discovery message type %T", msg) @@ -120,18 +123,18 @@ func (a *TargetApplier) processMessage(ctx context.Context, message core.Discove } // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly -func (a *TargetApplier) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { - if a.activeSnapshot == nil { - a.startNewSnapshot(chunk, logger) +func (c *TargetHandler) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { + if c.activeSnapshot == nil { + c.startNewSnapshot(chunk, logger) return nil } - snapshot := a.activeSnapshot + snapshot := c.activeSnapshot // Check if a new snapshot arrived if snapshot.snapshotID != chunk.SnapshotID { // If current snapshot is complete apply it first if snapshot.complete { - if err := a.applySnapshot(ctx, snapshot, logger); err != nil { + if err := c.applySnapshot(ctx, snapshot, logger); err != nil { return err } } else { @@ -144,40 +147,40 @@ func (a *TargetApplier) processSnapshot(ctx context.Context, chunk core.Discover } // Start collecting the new snapshot - a.startNewSnapshot(chunk, logger) + c.startNewSnapshot(chunk, logger) return nil } - return a.collectSnapshot(chunk, logger) + return c.collectSnapshot(chunk, logger) } -func (a *TargetApplier) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { - a.activeSnapshot = &snapshotBuffer{ +func (c *TargetHandler) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { + c.activeSnapshot = &snapshotBuffer{ snapshotID: chunk.SnapshotID, totalChunks: chunk.TotalChunks, received: make(map[int][]core.DiscoveredTarget), complete: false, } // Delete buffered events that will be current with new snapshot - a.deferredEvents = nil + c.deferredEvents = nil - a.collectSnapshot(chunk, logger) + c.collectSnapshot(chunk, logger) } -func (a *TargetApplier) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { - snapshot := a.activeSnapshot +func (c *TargetHandler) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { + snapshot := c.activeSnapshot if chunk.TotalChunks != snapshot.totalChunks { logger.Error(nil, "snapshot totalChunks mismatch", "snapshotID", snapshot.snapshotID) } if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks { logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex) - a.activeSnapshot = nil + c.activeSnapshot = nil return nil } if _, exists := snapshot.received[chunk.ChunkIndex]; exists { logger.Error(nil, "duplicate snapshot chunk", "index", chunk.ChunkIndex) - a.activeSnapshot = nil + c.activeSnapshot = nil return nil } @@ -190,10 +193,10 @@ func (a *TargetApplier) collectSnapshot(chunk core.DiscoverySnapshot, logger log return nil } -func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { +func (c *TargetHandler) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { select { case <-ctx.Done(): - a.activeSnapshot = nil + c.activeSnapshot = nil return nil default: } @@ -202,7 +205,7 @@ func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuf for i := 0; i < snapshot.totalChunks; i++ { select { case <-ctx.Done(): - a.activeSnapshot = nil + c.activeSnapshot = nil return nil default: } @@ -210,7 +213,7 @@ func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuf chunk, ok := snapshot.received[i] if !ok { logger.Error(nil, "missing snapshot chunk", "index", i) - a.activeSnapshot = nil + c.activeSnapshot = nil return nil } allTargets = append(allTargets, chunk...) @@ -226,34 +229,34 @@ func (a *TargetApplier) applySnapshot(ctx context.Context, snapshot *snapshotBuf // a.applyTargets // Replay deferred events - for _, event := range a.deferredEvents { + for _, event := range c.deferredEvents { select { case <-ctx.Done(): return nil default: } - if err := a.applyEvent(ctx, event, logger); err != nil { + if err := c.applyEvent(ctx, event, logger); err != nil { return err } } - a.activeSnapshot = nil - a.deferredEvents = nil + c.activeSnapshot = nil + c.deferredEvents = nil return nil } -func (a *TargetApplier) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (c *TargetHandler) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { // If snapshot collecting is active defer events - if a.activeSnapshot != nil { - a.deferredEvents = append(a.deferredEvents, event) + if c.activeSnapshot != nil { + c.deferredEvents = append(c.deferredEvents, event) return nil } // Apply events - return a.applyEvent(ctx, event, logger) + return c.applyEvent(ctx, event, logger) } -func (a *TargetApplier) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (c *TargetHandler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { switch event.Event { case core.CREATE: logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) @@ -264,3 +267,19 @@ func (a *TargetApplier) applyEvent(ctx context.Context, event core.DiscoveryEven } return nil } + +func (c *TargetHandler) fetchExistingTargets() ([]gnmicv1alpha1.Target, error) { + var targetList gnmicv1alpha1.TargetList + + err := c.client.List(c.ctx, &targetList, + client.InNamespace(c.targetSource.Namespace), + client.MatchingLabels{ + "gnmic.io/source": c.targetSource.Name, + }, + ) + if err != nil { + return nil, err + } + + return targetList.Items, nil +} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 77a3a35..4d5f400 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -164,7 +164,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour // startDiscoveryPipeline creates and starts a discover pipeline for a TargetSource // // Pipeline semantics: -// 1. target-applier is mandatory and must start first +// 1. target-handler is mandatory and must start first // 2. loader is optional and conditional on spec // 3. Permanent failure of required components shuts down the pipeline // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister @@ -176,30 +176,30 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName return err } - // Create target applier instance - applier := discovery.NewTargetApplier( + // Create target targetHandler instance + targetHandler := discovery.NewTargetHandler( r.Client, r.Scheme, targetSource, targetChannel, ) - // Start target applier - applierReady := make(chan struct{}) + // Start target handler + handlerReady := make(chan struct{}) supervisor.StartSupervisedComponent(discovery.ComponentSpec{ - Name: "target-applier", + Name: "target-handler", Policy: discovery.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, EscalatesOnFailure: true, Run: func(ctx context.Context) error { - close(applierReady) // Signals that applier started successfully - return applier.Run(ctx) + close(handlerReady) // Signals that handler started successfully + return targetHandler.Run(ctx) }, }) - // Wait for applier to be ready before starting loader + // Wait for handler to be ready before starting loader select { - case <-applierReady: + case <-handlerReady: case <-supervisor.Done(): return nil } From 240a2bc382c5133829d327bde1cebfb4fd1530e9 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Sun, 26 Apr 2026 19:59:29 -0600 Subject: [PATCH 054/110] ran go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f236ded..827da2a 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.25.5 require ( github.com/cert-manager/cert-manager v1.19.3 github.com/go-logr/logr v1.4.3 + github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.27.3 github.com/onsi/gomega v1.38.3 github.com/openconfig/gnmic/pkg/api v0.1.10 @@ -47,7 +48,6 @@ require ( github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20251213031049-b05bdaca462f // indirect - github.com/google/uuid v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect From 80d321129925fa53d4374a789c821e76bbe82d95 Mon Sep 17 00:00:00 2001 From: Janooski Date: Mon, 27 Apr 2026 08:55:33 +0000 Subject: [PATCH 055/110] manual implementation of required fields --- go.mod | 2 +- internal/apiserver/apiserver.go | 35 +++++++++++++++++++++++---------- internal/apiserver/gen.go | 31 +++++++++++++++-------------- internal/apiserver/openapi.yaml | 10 ++++++++++ internal/apiserver/temp.md | 9 +++++++-- 5 files changed, 59 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 0a1d66c..8e56473 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/gnmic/operator go 1.25.5 require ( + github.com/bytedance/gopkg v0.1.3 github.com/cert-manager/cert-manager v1.19.3 github.com/getkin/kin-openapi v0.133.0 github.com/go-logr/logr v1.4.3 @@ -19,7 +20,6 @@ require ( ) require ( - github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index b24c7d7..5865b86 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -9,6 +9,7 @@ import ( "fmt" "net/http" + "github.com/bytedance/gopkg/util/logger" "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery/core" @@ -53,20 +54,35 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { // CreateTargets binds payload to payloadTargets struct defined in openapi contract. Passes func (a *APIServer) CreateTargets(c *gin.Context) { - var payloadTargets Targets fmt.Println("Binding Target to PayloadTarget") if err := c.ShouldBind(&payloadTargets); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - fmt.Printf("err: %s", err.Error) return } + // the openapi.yaml contract has required fields, but these are not enforced... To enforce them, a middleware + // needs to be used: https://deepwiki.com/oapi-codegen/oapi-codegen/7-middleware-and-validation + // The one for gin-gonic is not actively maintained, so for v1 I'll do validation manually. To be improved. + if payloadTargets.TargetSourceNameSpace == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "targetSourceNameSpace is required"}) + return + } + if payloadTargets.TargetSourceName == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "targetSourceName is required"}) + return + } + + targets := []core.DiscoveryEvent{} - if len(*payloadTargets.TargetList) > 0 { // doesn't work on empty TargetList - for _, target := range *payloadTargets.TargetList { + if len(payloadTargets.TargetList) > 0 { + for i, target := range payloadTargets.TargetList { + if target.Address == "" || target.Name == "" || target.Operation == "" { + logger.Warn("Target receieved at index %s by pull interface does not contain Address, Name or Operation and is skipped.", i) + break + } event := core.CREATE - switch *target.Operation { + switch target.Operation { case Create: event = core.CREATE case Delete: @@ -74,8 +90,8 @@ func (a *APIServer) CreateTargets(c *gin.Context) { } targets = append(targets, core.DiscoveryEvent{ Target: core.DiscoveredTarget{ - Name: *target.Name, - Address: *target.Address, + Name: target.Name, + Address: target.Address, Labels: map[string]string{"key": "Is this a tag?"}, }, Event: event, @@ -84,8 +100,8 @@ func (a *APIServer) CreateTargets(c *gin.Context) { } key := types.NamespacedName{ - Namespace: *payloadTargets.TargetSourceNameSpace, - Name: *payloadTargets.TargetSourceName, + Namespace: payloadTargets.TargetSourceNameSpace, + Name: payloadTargets.TargetSourceName, } ch, ok := a.DiscoveryRegistry.Get(key) if !ok { @@ -93,7 +109,6 @@ func (a *APIServer) CreateTargets(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": "Target Source doesn't exist"}) return } - fmt.Println("Preparing SendEvents") core.SendEvents(context.Background(), ch, targets, 10) // make number constant c.JSON(http.StatusOK, payloadTargets) } diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index 84321bd..ae7704c 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -36,11 +36,11 @@ func (e TargetOperation) Valid() bool { // Target defines model for Target. type Target struct { - Address *string `json:"address,omitempty"` - Name *string `json:"name,omitempty"` - Operation *TargetOperation `json:"operation,omitempty"` - Profile *string `json:"profile,omitempty"` - Tags *[]string `json:"tags,omitempty"` + Address string `json:"address"` + Name string `json:"name"` + Operation TargetOperation `json:"operation"` + Profile *string `json:"profile,omitempty"` + Tags *[]string `json:"tags,omitempty"` } // TargetOperation defines model for Target.Operation. @@ -48,9 +48,9 @@ type TargetOperation string // Targets defines model for Targets. type Targets struct { - TargetList *[]Target `json:"TargetList,omitempty"` - TargetSourceName *string `json:"TargetSourceName,omitempty"` - TargetSourceNameSpace *string `json:"targetSourceNameSpace,omitempty"` + TargetList []Target `json:"TargetList"` + TargetSourceName string `json:"TargetSourceName"` + TargetSourceNameSpace string `json:"targetSourceNameSpace"` } // ServerInterface represents all server handlers. @@ -132,13 +132,14 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/4yTz2vdMAzH/xWj7RiStLvlNsooD7au7PU2dvAcJc8lsY2kDB4l//uwnf5Ikwc9xchf", - "Wfp+pDyB8WPwDp0wNE/A5oSjTscHTT1KPAXyAUksprhuW0JORzkHhAZYyLoe5gKcHnH3Ij6gxXoXb9FN", - "IzS/wRBqQSigxQEF4U+xTQzkOzvsPyq6T21YwXG/nyWgifQZ5teA//uIRqIiu+StzXzx3bKsSnwm7KCB", - "T9Urt2qBVi3ENnWfqxz9RAbvLjGSd6Jj0GZPubURQ9Z1PomtRFzQ3/04GPUzcfekfn07Pqiv9wco4B8S", - "p0lAXdbl1TIep4OFBr6U12UNBQQtp2S4ylN6y8lnJi8zPbTQwM1KVgAhB+84w7yur+LHeCfoUrIOYbAm", - "pVePnPcic7w0ig/i50uAWmRDNuQlfH5TZXet4skYZO6mYcibwtM4ajq/OFOyZFin5IRqzTelVGHQycjy", - "36wB3aLcDBML0n2UbQjV8bNu8o1eEcpEDtt3zd2iKJNlKpWf53n+HwAA////AVrz1gMAAA==", + "H4sIAAAAAAAC/4yTz4rbMBDGX0VMezSxd3vzrSxlCbTbpdlb6UGVJ4kWW1JnxoWw+N2LJCdrxw70ZCHN", + "n29+3/gNjO+Cd+iEoX4DNkfsdDq+aDqgxFMgH5DEYrrXTUPI6SingFADC1l3gKEApztcfYgFtFjv4iu6", + "voP6JxhCLQgFNNiiIPwqlomB/N6260VFH5IMK9it6xkvNJE+wTAUQPint4RNbJ/EFpd5piLflfjfr2gk", + "1so8eAkkP3y1LDMxHwn3UMOH8p1wOeItR7YLhecuO9+TwadbNOUqaBe0WYu8Gng9baVnMZ1pySLWtW7v", + "U0cr0R04PH3bGvU9EfSkfnzZvajPz1so4C8SJ+Oh2lSbu3EbnA4Wavi0ud9UUEDQckzUyrwUU9g+g724", + "s22ghodZWJyTg3ecHbmv7uLHeCfoUrIOobUmpZevnNcwm3HLz//0kBeUz/fr3BpkQzbkX+Ecq/LQjeLe", + "GGTe922b95X7rtN0ugysZMywTskR1Rx7SilDq9N849875/aI8tD2LEjPMWwBroqfuchJvCKUnhw2V+Ie", + "UZTJYSq1H4Zh+BcAAP//nMyhxFwEAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index d84f265..38917ee 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -21,6 +21,8 @@ paths: application/json: schema: type: object + required: + - Targets properties: Targets: $ref: '#/components/schemas/Targets' @@ -28,6 +30,10 @@ components: schemas: Targets: type: object + required: + - targetSourceNameSpace + - TargetSourceName + - TargetList properties: targetSourceNameSpace: type: string @@ -39,6 +45,10 @@ components: $ref: '#/components/schemas/Target' Target: type: object + required: + - name + - address + - operation properties: name: type: string diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index 90f3fb0..7bca5d8 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -1,6 +1,11 @@ ## CURL request curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' - ## Empty TargetList -curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' \ No newline at end of file +curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' + +## Empty Target in Target List +curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":""}]}' + +## Empty TargetSourceName +curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create"}]}' \ No newline at end of file From b2d3c189efa2a8cab570708f63c0588085481673 Mon Sep 17 00:00:00 2001 From: Janooski Date: Mon, 27 Apr 2026 14:26:32 +0000 Subject: [PATCH 056/110] add URL templating --- cmd/main.go | 3 +- internal/apiserver/apiserver.go | 50 +++++++++++++++++++++++++-------- internal/apiserver/temp.md | 27 +++++++++++++++--- 3 files changed, 62 insertions(+), 18 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 5db5f6e..125e32d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -233,12 +233,11 @@ func main() { } if apiAddr != "" { - api, err := apiserver.New(apiAddr, clusterReconciler) + api, err := apiserver.New(apiAddr, clusterReconciler, discoveryChunkSize) if err != nil { setupLog.Error(err, "unable to intialize gin API server") os.Exit(1) } - // api.ChunkSize = discoveryChunkSize api.DiscoveryRegistry = discoveryRegistry err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 5865b86..1e75a20 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -6,7 +6,6 @@ package apiserver import ( "context" - "fmt" "net/http" "github.com/bytedance/gopkg/util/logger" @@ -22,9 +21,15 @@ type APIServer struct { router *gin.Engine clusterReconciler *controller.ClusterReconciler DiscoveryRegistry *registry.Registry[types.NamespacedName, []core.DiscoveryMessage] + chunkSize int } -func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServer, error) { +type urlStruct struct { + namespace string `uri:"namespace" binding:"required"` + gNMIcClusterName string `uri:"gNMIcClusterName" binding:"required"` +} + +func New(addr string, clusterReconciler *controller.ClusterReconciler, chunkSize int) (*APIServer, error) { router := gin.Default() a := &APIServer{ Server: &http.Server{ @@ -33,9 +38,9 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServ }, router: router, clusterReconciler: clusterReconciler, + chunkSize: chunkSize, } - - apiBaseURL := "/api/v1/namespaceCluster/namegNMIcCluster" + apiBaseURL := "/api/v1/:namespace/:gNMIcClusterName" RegisterHandlersWithOptions(router, a, GinServerOptions{BaseURL: apiBaseURL}) return a, nil } @@ -44,7 +49,8 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler) (*APIServ // GetClusterPlan returns cluster plan func (a *APIServer) GetClusterPlan(c *gin.Context) { - plan, err := a.clusterReconciler.GetClusterPlan("temp", "temp") + url := parseURI(c) + plan, err := a.clusterReconciler.GetClusterPlan(url.namespace, url.gNMIcClusterName) if err != nil { c.String(404, err.Error()) return @@ -52,10 +58,15 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { c.JSON(200, plan) } -// CreateTargets binds payload to payloadTargets struct defined in openapi contract. Passes +// CreateTargets binds payload to payloadTargets struct defined in openapi contract. Creates a []core.DiscoveryEvent sends it to the core package. func (a *APIServer) CreateTargets(c *gin.Context) { + // Discussion with Daniel: this was input from Jan and Karim that the URI should be a template + // But I don't think it is needed in the CreateTargets function + // url := parseURI(c) + // fmt.Printf("namespace: %s", url.namespace) + // fmt.Printf("gNMIcClusterName: %s", url.gNMIcClusterName) + var payloadTargets Targets - fmt.Println("Binding Target to PayloadTarget") if err := c.ShouldBind(&payloadTargets); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return @@ -73,14 +84,18 @@ func (a *APIServer) CreateTargets(c *gin.Context) { return } - targets := []core.DiscoveryEvent{} if len(payloadTargets.TargetList) > 0 { for i, target := range payloadTargets.TargetList { if target.Address == "" || target.Name == "" || target.Operation == "" { - logger.Warn("Target receieved at index %s by pull interface does not contain Address, Name or Operation and is skipped.", i) + logger.Warn("Target receieved at index", i , " by pull interface does not contain Address, Name or Operation and is skipped.") + break + } + if target.Operation.Valid() != true { + logger.Warn("Target receieved at index", i , " by pull interface has invalid Operation.") break } + event := core.CREATE switch target.Operation { case Create: @@ -88,6 +103,7 @@ func (a *APIServer) CreateTargets(c *gin.Context) { case Delete: event = core.DELETE } + targets = append(targets, core.DiscoveryEvent{ Target: core.DiscoveredTarget{ Name: target.Name, @@ -105,10 +121,20 @@ func (a *APIServer) CreateTargets(c *gin.Context) { } ch, ok := a.DiscoveryRegistry.Get(key) if !ok { - // Error message to be udpated!! - c.JSON(http.StatusBadRequest, gin.H{"error": "Target Source doesn't exist"}) + logger.Error("TargetSource " , payloadTargets.TargetSourceNameSpace, "/", payloadTargets.TargetSourceName, "does not exist.") + c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource does not exist"}) return } - core.SendEvents(context.Background(), ch, targets, 10) // make number constant + core.SendEvents(context.Background(), ch, targets, a.chunkSize) c.JSON(http.StatusOK, payloadTargets) } + +// parseURI parses URI to urlStruct. +func parseURI(c *gin.Context) (url urlStruct) { + var u urlStruct + if err := c.ShouldBindUri(&u); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + return u +} diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index 7bca5d8..fae3f49 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -1,11 +1,30 @@ ## CURL request -curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' +curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' ## Empty TargetList -curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' +curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' ## Empty Target in Target List -curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":""}]}' +curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":""}]}' ## Empty TargetSourceName -curl -X POST "http://localhost:8082/api/v1/namespaceCluster/namegNMIcCluster/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create"}]}' \ No newline at end of file +curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create"}]}' + +## Wrong operation +curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"notupdate","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' + + +http://gnmic-controller-manager-api.gnmic-system.svc.cluster.local:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets +{ + "TargetSourceName": "netbox", + "TargetSourceNameSpace": "netbox", + "TargetList": [ + { + "name": "{{ data.name }}", + "address": "{{ data.primary_ip4.address.split('/')[0] if data.primary_ip4 else '' }}:{{ data.custom_fields.port }}", + "profile": "{{ data.custom_fields.profile | default('') }}", + "tags": {{ data.tags | map(attribute='name') | list | tojson }}, + "operation":"create" + } + ] +} From 7ef1281a7b37bd8b9a845501f7011c615710429b Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 27 Apr 2026 10:10:08 -0600 Subject: [PATCH 057/110] renamed target applier to message processor & created client.go for generic functions --- internal/controller/discovery/client.go | 25 ++++ ...target_handler.go => message_processor.go} | 112 ++++++++---------- .../controller/targetsource_controller.go | 2 +- 3 files changed, 74 insertions(+), 65 deletions(-) create mode 100644 internal/controller/discovery/client.go rename internal/controller/discovery/{target_handler.go => message_processor.go} (63%) diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go new file mode 100644 index 0000000..72147b7 --- /dev/null +++ b/internal/controller/discovery/client.go @@ -0,0 +1,25 @@ +package discovery + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" +) + +func fetchExistingTargets(ctx context.Context, c client.Client, ts *gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { + var targetList gnmicv1alpha1.TargetList + + err := c.List(ctx, &targetList, + client.InNamespace(ts.Namespace), + client.MatchingLabels{ + "gnmic.io/source": ts.Name, + }, + ) + if err != nil { + return nil, err + } + + return targetList.Items, nil +} diff --git a/internal/controller/discovery/target_handler.go b/internal/controller/discovery/message_processor.go similarity index 63% rename from internal/controller/discovery/target_handler.go rename to internal/controller/discovery/message_processor.go index e8c0308..65c8b44 100644 --- a/internal/controller/discovery/target_handler.go +++ b/internal/controller/discovery/message_processor.go @@ -20,8 +20,8 @@ type snapshotBuffer struct { complete bool } -// TargetHandler consumes discovered targets and applies them to Kubernetes -type TargetHandler struct { +// MessageProcessor consumes discovered targets and applies them to Kubernetes +type MessageProcessor struct { ctx context.Context client client.Client scheme *runtime.Scheme @@ -33,9 +33,9 @@ type TargetHandler struct { deferredEvents []core.DiscoveryEvent } -// NewTargetHandler wires a TargetHandler instance -func NewTargetHandler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetHandler { - return &TargetHandler{ +// NewMessageProcessor wires a MessageProcessor instance +func NewMessageProcessor(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *MessageProcessor { + return &MessageProcessor{ client: c, scheme: s, targetSource: ts, @@ -45,40 +45,40 @@ func NewTargetHandler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.Targ // Run is a long‑running loop that receives target snapshots // and reconciles Target CRs accordingly -func (c *TargetHandler) Run(ctx context.Context) error { - c.ctx = ctx +func (m *MessageProcessor) Run(ctx context.Context) error { + m.ctx = ctx - logger := log.FromContext(c.ctx). + logger := log.FromContext(m.ctx). WithValues( - "name", c.targetSource.Name, - "namespace", c.targetSource.Namespace, + "name", m.targetSource.Name, + "namespace", m.targetSource.Namespace, ) logger.Info("target handler started") - for c.ctx.Err() == nil { + for m.ctx.Err() == nil { select { - case batch, ok := <-c.in: + case batch, ok := <-m.in: if !ok { // Channel closed, pipeline is shutting down logger.Info("input channel closed, stopping target handler") return nil } - c.queue = append(c.queue, batch...) + m.queue = append(m.queue, batch...) case <-ctx.Done(): logger.Info("context canceled, stopping target handler") return nil } - for len(c.queue) > 0 { + for len(m.queue) > 0 { if ctx.Err() != nil { return nil // why return nil? } - msg := c.queue[0] - c.queue = c.queue[1:] + msg := m.queue[0] + m.queue = m.queue[1:] - if err := c.processMessage(c.ctx, msg, logger); err != nil { + if err := m.processMessage(m.ctx, msg, logger); err != nil { // Returning error lets the supervisor (controller) // tear down and restart the pipeline via reconciliation // Q: when to return an error vs just log and continue? @@ -92,7 +92,7 @@ func (c *TargetHandler) Run(ctx context.Context) error { return nil } -func (c *TargetHandler) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { +func (m *MessageProcessor) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { if err := ctx.Err(); err != nil { return err } @@ -107,7 +107,7 @@ func (c *TargetHandler) processMessage(ctx context.Context, message core.Discove "index", msg.ChunkIndex, "targetCount", len(msg.Targets), ) - return c.processSnapshot(ctx, msg, logger) + return m.processSnapshot(ctx, msg, logger) case core.DiscoveryEvent: // Process individual event-driven update @@ -115,7 +115,7 @@ func (c *TargetHandler) processMessage(ctx context.Context, message core.Discove "received discovery event", "target", msg.Target.Name, ) - return c.processEvent(ctx, msg, logger) + return m.processEvent(ctx, msg, logger) default: return fmt.Errorf("unknonw discovery message type %T", msg) @@ -123,18 +123,18 @@ func (c *TargetHandler) processMessage(ctx context.Context, message core.Discove } // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly -func (c *TargetHandler) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { - if c.activeSnapshot == nil { - c.startNewSnapshot(chunk, logger) +func (m *MessageProcessor) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { + if m.activeSnapshot == nil { + m.startNewSnapshot(chunk, logger) return nil } - snapshot := c.activeSnapshot + snapshot := m.activeSnapshot // Check if a new snapshot arrived if snapshot.snapshotID != chunk.SnapshotID { // If current snapshot is complete apply it first if snapshot.complete { - if err := c.applySnapshot(ctx, snapshot, logger); err != nil { + if err := m.applySnapshot(ctx, snapshot, logger); err != nil { return err } } else { @@ -147,40 +147,40 @@ func (c *TargetHandler) processSnapshot(ctx context.Context, chunk core.Discover } // Start collecting the new snapshot - c.startNewSnapshot(chunk, logger) + m.startNewSnapshot(chunk, logger) return nil } - return c.collectSnapshot(chunk, logger) + return m.collectSnapshot(chunk, logger) } -func (c *TargetHandler) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { - c.activeSnapshot = &snapshotBuffer{ +func (m *MessageProcessor) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { + m.activeSnapshot = &snapshotBuffer{ snapshotID: chunk.SnapshotID, totalChunks: chunk.TotalChunks, received: make(map[int][]core.DiscoveredTarget), complete: false, } // Delete buffered events that will be current with new snapshot - c.deferredEvents = nil + m.deferredEvents = nil - c.collectSnapshot(chunk, logger) + m.collectSnapshot(chunk, logger) } -func (c *TargetHandler) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { - snapshot := c.activeSnapshot +func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { + snapshot := m.activeSnapshot if chunk.TotalChunks != snapshot.totalChunks { logger.Error(nil, "snapshot totalChunks mismatch", "snapshotID", snapshot.snapshotID) } if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks { logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex) - c.activeSnapshot = nil + m.activeSnapshot = nil return nil } if _, exists := snapshot.received[chunk.ChunkIndex]; exists { logger.Error(nil, "duplicate snapshot chunk", "index", chunk.ChunkIndex) - c.activeSnapshot = nil + m.activeSnapshot = nil return nil } @@ -193,10 +193,10 @@ func (c *TargetHandler) collectSnapshot(chunk core.DiscoverySnapshot, logger log return nil } -func (c *TargetHandler) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { +func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { select { case <-ctx.Done(): - c.activeSnapshot = nil + m.activeSnapshot = nil return nil default: } @@ -205,7 +205,7 @@ func (c *TargetHandler) applySnapshot(ctx context.Context, snapshot *snapshotBuf for i := 0; i < snapshot.totalChunks; i++ { select { case <-ctx.Done(): - c.activeSnapshot = nil + m.activeSnapshot = nil return nil default: } @@ -213,7 +213,7 @@ func (c *TargetHandler) applySnapshot(ctx context.Context, snapshot *snapshotBuf chunk, ok := snapshot.received[i] if !ok { logger.Error(nil, "missing snapshot chunk", "index", i) - c.activeSnapshot = nil + m.activeSnapshot = nil return nil } allTargets = append(allTargets, chunk...) @@ -229,34 +229,34 @@ func (c *TargetHandler) applySnapshot(ctx context.Context, snapshot *snapshotBuf // a.applyTargets // Replay deferred events - for _, event := range c.deferredEvents { + for _, event := range m.deferredEvents { select { case <-ctx.Done(): return nil default: } - if err := c.applyEvent(ctx, event, logger); err != nil { + if err := m.applyEvent(ctx, event, logger); err != nil { return err } } - c.activeSnapshot = nil - c.deferredEvents = nil + m.activeSnapshot = nil + m.deferredEvents = nil return nil } -func (c *TargetHandler) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (m *MessageProcessor) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { // If snapshot collecting is active defer events - if c.activeSnapshot != nil { - c.deferredEvents = append(c.deferredEvents, event) + if m.activeSnapshot != nil { + m.deferredEvents = append(m.deferredEvents, event) return nil } // Apply events - return c.applyEvent(ctx, event, logger) + return m.applyEvent(ctx, event, logger) } -func (c *TargetHandler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { switch event.Event { case core.CREATE: logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) @@ -267,19 +267,3 @@ func (c *TargetHandler) applyEvent(ctx context.Context, event core.DiscoveryEven } return nil } - -func (c *TargetHandler) fetchExistingTargets() ([]gnmicv1alpha1.Target, error) { - var targetList gnmicv1alpha1.TargetList - - err := c.client.List(c.ctx, &targetList, - client.InNamespace(c.targetSource.Namespace), - client.MatchingLabels{ - "gnmic.io/source": c.targetSource.Name, - }, - ) - if err != nil { - return nil, err - } - - return targetList.Items, nil -} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 4d5f400..8070a3a 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -177,7 +177,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName } // Create target targetHandler instance - targetHandler := discovery.NewTargetHandler( + targetHandler := discovery.NewMessageProcessor( r.Client, r.Scheme, targetSource, From 7bcbcc023ff39e36a565e9235f503a98375f3327 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Fri, 24 Apr 2026 15:15:56 -0600 Subject: [PATCH 058/110] added const file for common labels --- internal/controller/discovery/client.go | 3 ++- internal/controller/discovery/core/const.go | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 internal/controller/discovery/core/const.go diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go index 3bc7ef7..d23c043 100644 --- a/internal/controller/discovery/client.go +++ b/internal/controller/discovery/client.go @@ -8,6 +8,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery/core" ) func FetchExistingTargets(ctx context.Context, c client.Client, ts gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { @@ -16,7 +17,7 @@ func FetchExistingTargets(ctx context.Context, c client.Client, ts gnmicv1alpha1 err := c.List(ctx, &targetList, client.InNamespace(ts.Namespace), client.MatchingLabels{ - "gnmic.io/source": ts.Name, + core.LabelTargetSourceName: ts.Name, }, ) if err != nil { diff --git a/internal/controller/discovery/core/const.go b/internal/controller/discovery/core/const.go new file mode 100644 index 0000000..82a5962 --- /dev/null +++ b/internal/controller/discovery/core/const.go @@ -0,0 +1,6 @@ +package core + +const ( + // Labels + LabelTargetSourceName = "operator.gnmic.dev/targetsource" +) From d10fc9ac868d50be64c123cbc619b2f4eb189682 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 27 Apr 2026 11:26:10 -0600 Subject: [PATCH 059/110] removed all package --- internal/controller/discovery/loaders/all/all.go | 5 ----- internal/controller/targetsource_controller.go | 1 - 2 files changed, 6 deletions(-) delete mode 100644 internal/controller/discovery/loaders/all/all.go diff --git a/internal/controller/discovery/loaders/all/all.go b/internal/controller/discovery/loaders/all/all.go deleted file mode 100644 index 3590cda..0000000 --- a/internal/controller/discovery/loaders/all/all.go +++ /dev/null @@ -1,5 +0,0 @@ -package all - -import ( - _ "github.com/gnmic/operator/internal/controller/discovery/loaders/http" -) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 8070a3a..49f9683 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -32,7 +32,6 @@ import ( "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/loaders" - _ "github.com/gnmic/operator/internal/controller/discovery/loaders/all" "github.com/gnmic/operator/internal/controller/discovery/registry" "github.com/go-logr/logr" ) From 108bd2dc3f58b2193535c8eadf6c30ee1d6d0dad Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Mon, 27 Apr 2026 11:34:11 -0600 Subject: [PATCH 060/110] changed error lookup to apierrors --- internal/controller/targetsource_controller.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 232c624..2f198a6 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -21,6 +21,7 @@ import ( "sync" "time" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -81,13 +82,12 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request WithValues("targetsource", req.NamespacedName) targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName) - if err != nil { - // If the TargetSource no longer exists, ensure runtime cleanup - if client.IgnoreNotFound(err) == nil { - logger.Info("TargetSource not found; stopping discovery pipeline") - r.stopDiscoveryPipeline(req.NamespacedName) - return ctrl.Result{}, nil - } + // If the TargetSource no longer exists, ensure runtime cleanup + if apierrors.IsNotFound(err) { + logger.Info("TargetSource not found; stopping discovery pipeline") + r.stopDiscoveryPipeline(req.NamespacedName) + return ctrl.Result{}, nil + } else if err != nil { return ctrl.Result{}, err } From b7dd0367e99a0c5435db00092c83e1bc01ab439b Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 29 Apr 2026 08:53:30 +0000 Subject: [PATCH 061/110] remove unused fiels --- internal/controller/discovery/mapper.go | 4 ---- internal/controller/discovery/mapper_test.go | 1 - 2 files changed, 5 deletions(-) delete mode 100644 internal/controller/discovery/mapper.go delete mode 100644 internal/controller/discovery/mapper_test.go diff --git a/internal/controller/discovery/mapper.go b/internal/controller/discovery/mapper.go deleted file mode 100644 index 18470b2..0000000 --- a/internal/controller/discovery/mapper.go +++ /dev/null @@ -1,4 +0,0 @@ -package discovery - -// This file makes diff between existing and new targets -// file decides which targets to create/update/delete diff --git a/internal/controller/discovery/mapper_test.go b/internal/controller/discovery/mapper_test.go deleted file mode 100644 index 5844159..0000000 --- a/internal/controller/discovery/mapper_test.go +++ /dev/null @@ -1 +0,0 @@ -package discovery From d3a9b5ca3021c9f0485698c1d1c54bbd3562bb9b Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 29 Apr 2026 11:56:56 +0000 Subject: [PATCH 062/110] rename files and restructure packages --- .../core/{loader_interface.go => loader.go} | 0 .../core/{message_interface.go => message.go} | 0 .../discovery/core/{helpers.go => send.go} | 0 internal/controller/discovery/core/types.go | 4 ++-- internal/controller/discovery/discovery.go | 17 +++++++++++++++++ .../loaders/{loaders.go => factory.go} | 0 .../loaders/http/{http.go => loader.go} | 0 .../http/{http_test.go => loader_test.go} | 0 .../discovery/{ => pipeline}/supervisor.go | 5 +++-- .../discovery/{ => reconciler}/client.go | 13 ++++++++++--- .../{ => reconciler}/message_processor.go | 2 +- internal/controller/targetsource_controller.go | 15 ++++++++------- 12 files changed, 41 insertions(+), 15 deletions(-) rename internal/controller/discovery/core/{loader_interface.go => loader.go} (100%) rename internal/controller/discovery/core/{message_interface.go => message.go} (100%) rename internal/controller/discovery/core/{helpers.go => send.go} (100%) create mode 100644 internal/controller/discovery/discovery.go rename internal/controller/discovery/loaders/{loaders.go => factory.go} (100%) rename internal/controller/discovery/loaders/http/{http.go => loader.go} (100%) rename internal/controller/discovery/loaders/http/{http_test.go => loader_test.go} (100%) rename internal/controller/discovery/{ => pipeline}/supervisor.go (95%) rename internal/controller/discovery/{ => reconciler}/client.go (68%) rename internal/controller/discovery/{ => reconciler}/message_processor.go (99%) diff --git a/internal/controller/discovery/core/loader_interface.go b/internal/controller/discovery/core/loader.go similarity index 100% rename from internal/controller/discovery/core/loader_interface.go rename to internal/controller/discovery/core/loader.go diff --git a/internal/controller/discovery/core/message_interface.go b/internal/controller/discovery/core/message.go similarity index 100% rename from internal/controller/discovery/core/message_interface.go rename to internal/controller/discovery/core/message.go diff --git a/internal/controller/discovery/core/helpers.go b/internal/controller/discovery/core/send.go similarity index 100% rename from internal/controller/discovery/core/helpers.go rename to internal/controller/discovery/core/send.go diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 3f6957a..28ec503 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -12,14 +12,14 @@ type DiscoveredTarget struct { Labels map[string]string } +type EventAction int + const ( DELETE EventAction = 0 CREATE EventAction = 1 UPDATE EventAction = 2 ) -type EventAction int - type DiscoveryEvent struct { Target DiscoveredTarget Event EventAction diff --git a/internal/controller/discovery/discovery.go b/internal/controller/discovery/discovery.go new file mode 100644 index 0000000..3dc51bd --- /dev/null +++ b/internal/controller/discovery/discovery.go @@ -0,0 +1,17 @@ +package discovery + +// Package discovery implements the discovery runtime subsystem. +// +// The discovery subsystem is responsible for: +// - Receiving discovery data from external providers (loaders, webhooks). +// - Supervising discovery pipelines and restart semantics. +// - Applying discovered state to Kubernetes Targets. +// +// The package is structured into the following subpackages: +// - core: message contracts, snapshot/event types, and transport helpers. +// - pipeline: supervision, restart policies, and lifecycle control. +// - reconciler: snapshot + event target state application logic. +// - loaders: target discovery providers (HTTP, webhook, etc.). +// - registry: key -> channel registry. +// +// At the moment, the targetsource controller imports specific subpackages explicitly. diff --git a/internal/controller/discovery/loaders/loaders.go b/internal/controller/discovery/loaders/factory.go similarity index 100% rename from internal/controller/discovery/loaders/loaders.go rename to internal/controller/discovery/loaders/factory.go diff --git a/internal/controller/discovery/loaders/http/http.go b/internal/controller/discovery/loaders/http/loader.go similarity index 100% rename from internal/controller/discovery/loaders/http/http.go rename to internal/controller/discovery/loaders/http/loader.go diff --git a/internal/controller/discovery/loaders/http/http_test.go b/internal/controller/discovery/loaders/http/loader_test.go similarity index 100% rename from internal/controller/discovery/loaders/http/http_test.go rename to internal/controller/discovery/loaders/http/loader_test.go diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/pipeline/supervisor.go similarity index 95% rename from internal/controller/discovery/supervisor.go rename to internal/controller/discovery/pipeline/supervisor.go index 710381e..042d305 100644 --- a/internal/controller/discovery/supervisor.go +++ b/internal/controller/discovery/pipeline/supervisor.go @@ -1,4 +1,4 @@ -package discovery +package pipeline import ( "context" @@ -25,12 +25,13 @@ type Supervisor struct { stopped bool } -// RestartPolicy defines the restart behavior for a component +// RestartPolicy defines restart behavior of a component type RestartPolicy struct { MaxRestarts int Backoff time.Duration } +// ComponentSpec defines a supervised component type ComponentSpec struct { Name string Run func(ctx context.Context) error diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/reconciler/client.go similarity index 68% rename from internal/controller/discovery/client.go rename to internal/controller/discovery/reconciler/client.go index 25100bd..4bbbbc1 100644 --- a/internal/controller/discovery/client.go +++ b/internal/controller/discovery/reconciler/client.go @@ -1,4 +1,4 @@ -package discovery +package reconciler import ( "context" @@ -9,10 +9,17 @@ import ( "github.com/gnmic/operator/internal/controller/discovery/core" ) -func fetchExistingTargets(ctx context.Context, c client.Client, ts *gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) { +func fetchExistingTargets( + ctx context.Context, + c client.Client, + ts *gnmicv1alpha1.TargetSource, +) ([]gnmicv1alpha1.Target, error) { + var targetList gnmicv1alpha1.TargetList - err := c.List(ctx, &targetList, + err := c.List( + ctx, + &targetList, client.InNamespace(ts.Namespace), client.MatchingLabels{ core.LabelTargetSourceName: ts.Name, diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/reconciler/message_processor.go similarity index 99% rename from internal/controller/discovery/message_processor.go rename to internal/controller/discovery/reconciler/message_processor.go index 65c8b44..0c205bd 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/reconciler/message_processor.go @@ -1,4 +1,4 @@ -package discovery +package reconciler import ( "context" diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 49f9683..35946d2 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -29,9 +29,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" - "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/loaders" + "github.com/gnmic/operator/internal/controller/discovery/pipeline" + "github.com/gnmic/operator/internal/controller/discovery/reconciler" "github.com/gnmic/operator/internal/controller/discovery/registry" "github.com/go-logr/logr" ) @@ -168,7 +169,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour // 3. Permanent failure of required components shuts down the pipeline // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error { - supervisor := discovery.NewSupervisor(context.Background()) + supervisor := pipeline.NewSupervisor(context.Background()) targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) if err := r.DiscoveryRegistry.Register(key, targetChannel); err != nil { @@ -176,7 +177,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName } // Create target targetHandler instance - targetHandler := discovery.NewMessageProcessor( + targetHandler := reconciler.NewMessageProcessor( r.Client, r.Scheme, targetSource, @@ -184,9 +185,9 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName ) // Start target handler handlerReady := make(chan struct{}) - supervisor.StartSupervisedComponent(discovery.ComponentSpec{ + supervisor.StartSupervisedComponent(pipeline.ComponentSpec{ Name: "target-handler", - Policy: discovery.RestartPolicy{ + Policy: pipeline.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, @@ -217,9 +218,9 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName return err } - supervisor.StartSupervisedComponent(discovery.ComponentSpec{ + supervisor.StartSupervisedComponent(pipeline.ComponentSpec{ Name: "loader", - Policy: discovery.RestartPolicy{ + Policy: pipeline.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, From 0c80394ab358c662fe519b872ed7219c2f7e384c Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 29 Apr 2026 12:01:40 +0000 Subject: [PATCH 063/110] rename target handler to target reconciler --- .../discovery/reconciler/message_processor.go | 8 ++++---- internal/controller/targetsource_controller.go | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/controller/discovery/reconciler/message_processor.go b/internal/controller/discovery/reconciler/message_processor.go index 0c205bd..2c4632c 100644 --- a/internal/controller/discovery/reconciler/message_processor.go +++ b/internal/controller/discovery/reconciler/message_processor.go @@ -53,20 +53,20 @@ func (m *MessageProcessor) Run(ctx context.Context) error { "name", m.targetSource.Name, "namespace", m.targetSource.Namespace, ) - logger.Info("target handler started") + logger.Info("target reconciler started") for m.ctx.Err() == nil { select { case batch, ok := <-m.in: if !ok { // Channel closed, pipeline is shutting down - logger.Info("input channel closed, stopping target handler") + logger.Info("input channel closed, stopping target reconciler") return nil } m.queue = append(m.queue, batch...) case <-ctx.Done(): - logger.Info("context canceled, stopping target handler") + logger.Info("context canceled, stopping target reconciler") return nil } @@ -88,7 +88,7 @@ func (m *MessageProcessor) Run(ctx context.Context) error { } } - logger.Info("target handler stopped") + logger.Info("target reconciler stopped") return nil } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 35946d2..6c9ad31 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -164,7 +164,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour // startDiscoveryPipeline creates and starts a discover pipeline for a TargetSource // // Pipeline semantics: -// 1. target-handler is mandatory and must start first +// 1. target reconciler is mandatory and must start first // 2. loader is optional and conditional on spec // 3. Permanent failure of required components shuts down the pipeline // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister @@ -176,14 +176,14 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName return err } - // Create target targetHandler instance - targetHandler := reconciler.NewMessageProcessor( + // Create target reconciler instance + targetReconciler := reconciler.NewMessageProcessor( r.Client, r.Scheme, targetSource, targetChannel, ) - // Start target handler + // Start target reconciler handlerReady := make(chan struct{}) supervisor.StartSupervisedComponent(pipeline.ComponentSpec{ Name: "target-handler", @@ -194,7 +194,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName EscalatesOnFailure: true, Run: func(ctx context.Context) error { close(handlerReady) // Signals that handler started successfully - return targetHandler.Run(ctx) + return targetReconciler.Run(ctx) }, }) // Wait for handler to be ready before starting loader From 04208bf078b170160a6ef72eda6b6ddaa3630070 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 29 Apr 2026 12:20:58 +0000 Subject: [PATCH 064/110] rename handler to reconciler --- internal/controller/targetsource_controller.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 6c9ad31..9078af2 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -184,22 +184,22 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName targetChannel, ) // Start target reconciler - handlerReady := make(chan struct{}) + reconcilerReady := make(chan struct{}) supervisor.StartSupervisedComponent(pipeline.ComponentSpec{ - Name: "target-handler", + Name: "target-reconciler", Policy: pipeline.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, EscalatesOnFailure: true, Run: func(ctx context.Context) error { - close(handlerReady) // Signals that handler started successfully + close(reconcilerReady) // Signals that reconciler started successfully return targetReconciler.Run(ctx) }, }) - // Wait for handler to be ready before starting loader + // Wait for reconciler to be ready before starting loader select { - case <-handlerReady: + case <-reconcilerReady: case <-supervisor.Done(): return nil } From c3818ce6f7693360496866d7ba1694f7ce702f32 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 29 Apr 2026 12:21:46 +0000 Subject: [PATCH 065/110] clarify interface files --- .../discovery/core/{loader.go => loader_interface.go} | 2 +- internal/controller/discovery/core/message.go | 4 ---- internal/controller/discovery/core/message_interface.go | 5 +++++ 3 files changed, 6 insertions(+), 5 deletions(-) rename internal/controller/discovery/core/{loader.go => loader_interface.go} (91%) create mode 100644 internal/controller/discovery/core/message_interface.go diff --git a/internal/controller/discovery/core/loader.go b/internal/controller/discovery/core/loader_interface.go similarity index 91% rename from internal/controller/discovery/core/loader.go rename to internal/controller/discovery/core/loader_interface.go index 8964be8..72f1898 100644 --- a/internal/controller/discovery/core/loader.go +++ b/internal/controller/discovery/core/loader_interface.go @@ -14,7 +14,7 @@ type Loader interface { Name() string // Start begins discovery and pushes target snapshots or events into the out channel - // The loader must stop cleanly when ctx is cancelled + // The loader must stop cleanly when ctx is canceled Start( ctx context.Context, targetsourceName types.NamespacedName, diff --git a/internal/controller/discovery/core/message.go b/internal/controller/discovery/core/message.go index 0836bc6..af4f6c1 100644 --- a/internal/controller/discovery/core/message.go +++ b/internal/controller/discovery/core/message.go @@ -1,8 +1,4 @@ package core -type DiscoveryMessage interface { - isDiscoveryMessage() -} - func (DiscoveryEvent) isDiscoveryMessage() {} func (DiscoverySnapshot) isDiscoveryMessage() {} diff --git a/internal/controller/discovery/core/message_interface.go b/internal/controller/discovery/core/message_interface.go new file mode 100644 index 0000000..07b819e --- /dev/null +++ b/internal/controller/discovery/core/message_interface.go @@ -0,0 +1,5 @@ +package core + +type DiscoveryMessage interface { + isDiscoveryMessage() +} From ab09c7c6c76ea36b27d89049e9d45953fd905801 Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 29 Apr 2026 12:23:07 +0000 Subject: [PATCH 066/110] small code refactor --- internal/apiserver/apiserver.go | 57 +++++++++++++++++---------------- internal/apiserver/gen.go | 27 +++++++++------- internal/apiserver/openapi.yaml | 5 +-- internal/apiserver/temp.md | 14 ++++---- 4 files changed, 55 insertions(+), 48 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 1e75a20..b6f04e8 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -6,6 +6,7 @@ package apiserver import ( "context" + "fmt" "net/http" "github.com/bytedance/gopkg/util/logger" @@ -25,8 +26,8 @@ type APIServer struct { } type urlStruct struct { - namespace string `uri:"namespace" binding:"required"` - gNMIcClusterName string `uri:"gNMIcClusterName" binding:"required"` + namespace string `uri:"namespace" binding:"required"` + gNMIcControllerName string `uri:"gNMIcControllerName" binding:"required"` } func New(addr string, clusterReconciler *controller.ClusterReconciler, chunkSize int) (*APIServer, error) { @@ -40,7 +41,7 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler, chunkSize clusterReconciler: clusterReconciler, chunkSize: chunkSize, } - apiBaseURL := "/api/v1/:namespace/:gNMIcClusterName" + apiBaseURL := "/api/v1/:namespace/:gNMIcControllerName" RegisterHandlersWithOptions(router, a, GinServerOptions{BaseURL: apiBaseURL}) return a, nil } @@ -50,7 +51,7 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler, chunkSize // GetClusterPlan returns cluster plan func (a *APIServer) GetClusterPlan(c *gin.Context) { url := parseURI(c) - plan, err := a.clusterReconciler.GetClusterPlan(url.namespace, url.gNMIcClusterName) + plan, err := a.clusterReconciler.GetClusterPlan(url.namespace, url.gNMIcControllerName) if err != nil { c.String(404, err.Error()) return @@ -60,12 +61,7 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { // CreateTargets binds payload to payloadTargets struct defined in openapi contract. Creates a []core.DiscoveryEvent sends it to the core package. func (a *APIServer) CreateTargets(c *gin.Context) { - // Discussion with Daniel: this was input from Jan and Karim that the URI should be a template - // But I don't think it is needed in the CreateTargets function - // url := parseURI(c) - // fmt.Printf("namespace: %s", url.namespace) - // fmt.Printf("gNMIcClusterName: %s", url.gNMIcClusterName) - + logger.Info("received POST request for CreateTargets.") var payloadTargets Targets if err := c.ShouldBind(&payloadTargets); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) @@ -76,32 +72,50 @@ func (a *APIServer) CreateTargets(c *gin.Context) { // needs to be used: https://deepwiki.com/oapi-codegen/oapi-codegen/7-middleware-and-validation // The one for gin-gonic is not actively maintained, so for v1 I'll do validation manually. To be improved. if payloadTargets.TargetSourceNameSpace == "" { + logger.Error("POST request does not contain value targetSourceNameSpace.") c.JSON(http.StatusBadRequest, gin.H{"error": "targetSourceNameSpace is required"}) return } if payloadTargets.TargetSourceName == "" { + logger.Error("POST request does not contain value targetSourceName.") c.JSON(http.StatusBadRequest, gin.H{"error": "targetSourceName is required"}) return } + key := types.NamespacedName{ + Namespace: payloadTargets.TargetSourceNameSpace, + Name: payloadTargets.TargetSourceName, + } + ch, ok := a.DiscoveryRegistry.Get(key) + if !ok { + logger.Error("TargetSource ", payloadTargets.TargetSourceNameSpace, "/", payloadTargets.TargetSourceName, "does not exist.") + c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource does not exist"}) + return + } + + fmt.Printf("payloadTargets %+v\n", payloadTargets) targets := []core.DiscoveryEvent{} if len(payloadTargets.TargetList) > 0 { for i, target := range payloadTargets.TargetList { if target.Address == "" || target.Name == "" || target.Operation == "" { - logger.Warn("Target receieved at index", i , " by pull interface does not contain Address, Name or Operation and is skipped.") + logger.Warn("Target receieved at index", i, " by pull interface does not contain Address, Name or Operation and is skipped.") break } if target.Operation.Valid() != true { - logger.Warn("Target receieved at index", i , " by pull interface has invalid Operation.") + logger.Warn("Target receieved at index", i, " by pull interface has invalid Operation.") break } event := core.CREATE switch target.Operation { - case Create: - event = core.CREATE - case Delete: + case Created: + event = core.UPDATE + case Updated: + event = core.UPDATE + case Deleted: event = core.DELETE + default: + logger.Warn("Received invalid Operation flag") } targets = append(targets, core.DiscoveryEvent{ @@ -114,17 +128,6 @@ func (a *APIServer) CreateTargets(c *gin.Context) { }) } } - - key := types.NamespacedName{ - Namespace: payloadTargets.TargetSourceNameSpace, - Name: payloadTargets.TargetSourceName, - } - ch, ok := a.DiscoveryRegistry.Get(key) - if !ok { - logger.Error("TargetSource " , payloadTargets.TargetSourceNameSpace, "/", payloadTargets.TargetSourceName, "does not exist.") - c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource does not exist"}) - return - } core.SendEvents(context.Background(), ch, targets, a.chunkSize) c.JSON(http.StatusOK, payloadTargets) } @@ -137,4 +140,4 @@ func parseURI(c *gin.Context) (url urlStruct) { return } return u -} +} \ No newline at end of file diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index ae7704c..11e23a2 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -18,16 +18,19 @@ import ( // Defines values for TargetOperation. const ( - Create TargetOperation = "create" - Delete TargetOperation = "delete" + Created TargetOperation = "created" + Deleted TargetOperation = "deleted" + Updated TargetOperation = "updated" ) // Valid indicates whether the value is a known member of the TargetOperation enum. func (e TargetOperation) Valid() bool { switch e { - case Create: + case Created: return true - case Delete: + case Deleted: + return true + case Updated: return true default: return false @@ -132,14 +135,14 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/4yTz4rbMBDGX0VMezSxd3vzrSxlCbTbpdlb6UGVJ4kWW1JnxoWw+N2LJCdrxw70ZCHN", - "n29+3/gNjO+Cd+iEoX4DNkfsdDq+aDqgxFMgH5DEYrrXTUPI6SingFADC1l3gKEApztcfYgFtFjv4iu6", - "voP6JxhCLQgFNNiiIPwqlomB/N6260VFH5IMK9it6xkvNJE+wTAUQPint4RNbJ/EFpd5piLflfjfr2gk", - "1so8eAkkP3y1LDMxHwn3UMOH8p1wOeItR7YLhecuO9+TwadbNOUqaBe0WYu8Gng9baVnMZ1pySLWtW7v", - "U0cr0R04PH3bGvU9EfSkfnzZvajPz1so4C8SJ+Oh2lSbu3EbnA4Wavi0ud9UUEDQckzUyrwUU9g+g724", - "s22ghodZWJyTg3ecHbmv7uLHeCfoUrIOobUmpZevnNcwm3HLz//0kBeUz/fr3BpkQzbkX+Ecq/LQjeLe", - "GGTe922b95X7rtN0ugysZMywTskR1Rx7SilDq9N849875/aI8tD2LEjPMWwBroqfuchJvCKUnhw2V+Ie", - "UZTJYSq1H4Zh+BcAAP//nMyhxFwEAAA=", + "H4sIAAAAAAAC/4yTzYrbMBDHX0VMezSxd3vzrSxlCbTbpdlb6UGVJokWW1JHo0JY/O5FkpO1Ywd60iDN", + "x39+M3oD5XrvLFoO0L5BUEfsZTZfJB2Qk+XJeSQ2mO+l1oQhm3zyCC0EJmMPMFRgZY+rDymBZONsekUb", + "e2h/giKUjBoqiF6PlsYOk/WrWibx5PamWy/A8pAlGcZ+Xdt4IYnkCYahAsI/0RDqJCULry69TQW/K3G/", + "X1FxylXYhCWc8vDVBJ6J+Ui4hxY+1O+06xF1PXJeKDxX2blICp9ukeUrp52Xas3zquH1sJWa1bSnJYuU", + "19i9yxUNp+nA4enbVonvmaAj8ePL7kV8ft5CBX+RQl4CaDbN5m7cDCu9gRY+be43DVTgJR8ztbosyBS2", + "K2Av09lqaOFh5pb6DN7ZUCZy39ylQznLaHOw9L4zKofXr6GsZBnGrXn+5wzDgvL5fp2bxqDI+PItzr5i", + "/BUiRKUwhH3surKvIfa9pNOlYcFjhLGCjyjm2HNI7TuZ+xt/8pzbI/JDFwMjPSe3BbgmHXORE39ByJEs", + "6itxj8hCFTeRyw/DMPwLAAD//2Qu5BBoBAAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index 38917ee..5accf2b 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -63,5 +63,6 @@ components: operation: type: string enum: - - create - - delete \ No newline at end of file + - created + - updated + - deleted \ No newline at end of file diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index fae3f49..715ef09 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -1,5 +1,5 @@ ## CURL request -curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' +curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"webhook-test", "TargetSourceNameSpace":"default", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' ## Empty TargetList curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' @@ -14,17 +14,17 @@ curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"notupdate","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' -http://gnmic-controller-manager-api.gnmic-system.svc.cluster.local:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets + { - "TargetSourceName": "netbox", - "TargetSourceNameSpace": "netbox", + "TargetSourceName": "webhook-test", + "TargetSourceNameSpace": "default", "TargetList": [ { "name": "{{ data.name }}", "address": "{{ data.primary_ip4.address.split('/')[0] if data.primary_ip4 else '' }}:{{ data.custom_fields.port }}", "profile": "{{ data.custom_fields.profile | default('') }}", - "tags": {{ data.tags | map(attribute='name') | list | tojson }}, - "operation":"create" + "tags": [{% for tag in data.tags %}"{{ tag.name }}"{% if not loop.last %}, {% endif %}{% endfor %}], + "operation": "{{ event }}" } ] -} +} \ No newline at end of file From e4df0d4a6245d71d48539414b0f3ab45136de874 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 29 Apr 2026 12:35:14 +0000 Subject: [PATCH 067/110] define EventAction to be go idomatic --- internal/controller/discovery/core/types.go | 20 +++++++++++-------- .../discovery/reconciler/message_processor.go | 6 +++--- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 28ec503..1ae2f7a 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -4,6 +4,18 @@ type LoaderConfig struct { ChunkSize int } +// EventAction represents the type of a discovery event +type EventAction int + +const ( + // EventDelete indicates that a target should be removed + EventDelete EventAction = iota + // EventCreate indicates that a target should be created + EventCreate + // EventUpdate indicates that a target should be updated + EventUpdate +) + // DiscoveredTarget represents a target discovered from an external source // before it is materialized as a Kubernetes Target CR type DiscoveredTarget struct { @@ -12,14 +24,6 @@ type DiscoveredTarget struct { Labels map[string]string } -type EventAction int - -const ( - DELETE EventAction = 0 - CREATE EventAction = 1 - UPDATE EventAction = 2 -) - type DiscoveryEvent struct { Target DiscoveredTarget Event EventAction diff --git a/internal/controller/discovery/reconciler/message_processor.go b/internal/controller/discovery/reconciler/message_processor.go index 2c4632c..a0e91e5 100644 --- a/internal/controller/discovery/reconciler/message_processor.go +++ b/internal/controller/discovery/reconciler/message_processor.go @@ -258,11 +258,11 @@ func (m *MessageProcessor) processEvent(ctx context.Context, event core.Discover func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { switch event.Event { - case core.CREATE: + case core.EventCreate: logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) - case core.UPDATE: + case core.EventUpdate: logger.Info("Would update target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) - case core.DELETE: + case core.EventDelete: logger.Info("Would delete target", "name", event.Target.Name) } return nil From 86c0af066faef2af3e75d68d3285c16dc6978bbe Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Wed, 29 Apr 2026 13:49:19 +0000 Subject: [PATCH 068/110] add webhook activation info to metadata of DiscoveryRegistry --- cmd/main.go | 2 +- internal/apiserver/apiserver.go | 2 +- internal/controller/discovery/core/types.go | 5 +++++ .../controller/discovery/registry/registry.go | 14 +++++++------- internal/controller/targetsource_controller.go | 18 +++++++++++------- 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index e4bad31..4cf6e94 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -86,7 +86,7 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - discoveryRegistry := registry.NewRegistry[types.NamespacedName, []core.DiscoveryMessage]() + discoveryRegistry := registry.NewRegistry[types.NamespacedName, core.DiscoveryRegistryValue]() mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 17e5c82..a7ca16a 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -14,7 +14,7 @@ type APIServer struct { Server *http.Server clusterReconciler *controller.ClusterReconciler - DiscoveryRegistry *registry.Registry[types.NamespacedName, []core.DiscoveryMessage] + DiscoveryRegistry *registry.Registry[types.NamespacedName, core.DiscoveryRegistryValue] } func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer { diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 1ae2f7a..68c9c7e 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -1,5 +1,10 @@ package core +type DiscoveryRegistryValue struct { + Channel chan<- []DiscoveryMessage + WebhookEnabled bool +} + type LoaderConfig struct { ChunkSize int } diff --git a/internal/controller/discovery/registry/registry.go b/internal/controller/discovery/registry/registry.go index 093bd2c..f2630e8 100644 --- a/internal/controller/discovery/registry/registry.go +++ b/internal/controller/discovery/registry/registry.go @@ -10,20 +10,20 @@ import ( // DO NOT USE a pointer type as K type Registry[K comparable, V any] struct { mu sync.RWMutex - m map[K]chan<- V + m map[K]V } func NewRegistry[K comparable, V any]() *Registry[K, V] { - return &Registry[K, V]{m: make(map[K]chan<- V)} + return &Registry[K, V]{m: make(map[K]V)} } -func (r *Registry[K, V]) Register(key K, ch chan<- V) error { +func (r *Registry[K, V]) Register(key K, value V) error { r.mu.Lock() defer r.mu.Unlock() if _, exists := r.m[key]; exists { return fmt.Errorf("already registered: %v", key) } - r.m[key] = ch + r.m[key] = value return nil } @@ -33,9 +33,9 @@ func (r *Registry[K, V]) Unregister(key K) { r.mu.Unlock() } -func (r *Registry[K, V]) Get(key K) (chan<- V, bool) { +func (r *Registry[K, V]) Get(key K) (V, bool) { r.mu.RLock() - ch, ok := r.m[key] + value, ok := r.m[key] r.mu.RUnlock() - return ch, ok + return value, ok } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 9078af2..c7e6460 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -66,7 +66,7 @@ type TargetSourceReconciler struct { BufferSize int ChunkSize int - DiscoveryRegistry *registry.Registry[types.NamespacedName, []core.DiscoveryMessage] + DiscoveryRegistry *registry.Registry[types.NamespacedName, core.DiscoveryRegistryValue] } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete @@ -108,7 +108,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } - logger.Info("Discover pipeline started") + logger.Info("Discovery pipeline started") return ctrl.Result{}, nil } @@ -161,7 +161,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour return nil } -// startDiscoveryPipeline creates and starts a discover pipeline for a TargetSource +// startDiscoveryPipeline creates and starts a discovery pipeline for a TargetSource // // Pipeline semantics: // 1. target reconciler is mandatory and must start first @@ -169,10 +169,16 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour // 3. Permanent failure of required components shuts down the pipeline // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error { + loaderConfigured := targetSource.Spec.Provider != nil + webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled + supervisor := pipeline.NewSupervisor(context.Background()) targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) - if err := r.DiscoveryRegistry.Register(key, targetChannel); err != nil { + if err := r.DiscoveryRegistry.Register(key, core.DiscoveryRegistryValue{ + Channel: targetChannel, + WebhookEnabled: webhookActivated, + }); err != nil { return err } @@ -205,8 +211,6 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName } // Create loader instance - loaderConfigured := targetSource.Spec.Provider != nil - webhookConfigured := targetSource.Spec.Webhook.Enabled != nil if loaderConfigured { loader, err := loaders.NewLoader( key, @@ -224,7 +228,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, - EscalatesOnFailure: !webhookConfigured, + EscalatesOnFailure: !webhookActivated, Run: func(ctx context.Context) error { return loader.Start(ctx, key, targetSource.Spec, targetChannel) }, From 284b1f290bd7f1c33f6213bba5399fb16ac0dae9 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 10:33:40 -0600 Subject: [PATCH 069/110] moved reconciler files to discovery --- internal/controller/discovery/{reconciler => }/client.go | 2 +- .../discovery/{reconciler => }/message_processor.go | 2 +- internal/controller/targetsource_controller.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename internal/controller/discovery/{reconciler => }/client.go (96%) rename internal/controller/discovery/{reconciler => }/message_processor.go (99%) diff --git a/internal/controller/discovery/reconciler/client.go b/internal/controller/discovery/client.go similarity index 96% rename from internal/controller/discovery/reconciler/client.go rename to internal/controller/discovery/client.go index 4bbbbc1..2deb477 100644 --- a/internal/controller/discovery/reconciler/client.go +++ b/internal/controller/discovery/client.go @@ -1,4 +1,4 @@ -package reconciler +package discovery import ( "context" diff --git a/internal/controller/discovery/reconciler/message_processor.go b/internal/controller/discovery/message_processor.go similarity index 99% rename from internal/controller/discovery/reconciler/message_processor.go rename to internal/controller/discovery/message_processor.go index a0e91e5..6e69c99 100644 --- a/internal/controller/discovery/reconciler/message_processor.go +++ b/internal/controller/discovery/message_processor.go @@ -1,4 +1,4 @@ -package reconciler +package discovery import ( "context" diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index c7e6460..84f9a6f 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -29,10 +29,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/loaders" "github.com/gnmic/operator/internal/controller/discovery/pipeline" - "github.com/gnmic/operator/internal/controller/discovery/reconciler" "github.com/gnmic/operator/internal/controller/discovery/registry" "github.com/go-logr/logr" ) @@ -183,7 +183,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName } // Create target reconciler instance - targetReconciler := reconciler.NewMessageProcessor( + targetReconciler := discovery.NewMessageProcessor( r.Client, r.Scheme, targetSource, From b59897c253b5db8858a03026ae187ac6c8959d19 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 10:34:55 -0600 Subject: [PATCH 070/110] renamed messageProcessor to targetReconciler --- ...sage_processor.go => target_reconciler.go} | 96 +++++++++---------- .../controller/targetsource_controller.go | 2 +- 2 files changed, 49 insertions(+), 49 deletions(-) rename internal/controller/discovery/{message_processor.go => target_reconciler.go} (72%) diff --git a/internal/controller/discovery/message_processor.go b/internal/controller/discovery/target_reconciler.go similarity index 72% rename from internal/controller/discovery/message_processor.go rename to internal/controller/discovery/target_reconciler.go index 6e69c99..4f3711c 100644 --- a/internal/controller/discovery/message_processor.go +++ b/internal/controller/discovery/target_reconciler.go @@ -20,8 +20,8 @@ type snapshotBuffer struct { complete bool } -// MessageProcessor consumes discovered targets and applies them to Kubernetes -type MessageProcessor struct { +// TargetReconciler consumes discovered targets and applies them to Kubernetes +type TargetReconciler struct { ctx context.Context client client.Client scheme *runtime.Scheme @@ -33,9 +33,9 @@ type MessageProcessor struct { deferredEvents []core.DiscoveryEvent } -// NewMessageProcessor wires a MessageProcessor instance -func NewMessageProcessor(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *MessageProcessor { - return &MessageProcessor{ +// NewTargetReconciler wires a TargetReconciler instance +func NewTargetReconciler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetReconciler { + return &TargetReconciler{ client: c, scheme: s, targetSource: ts, @@ -45,40 +45,40 @@ func NewMessageProcessor(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.T // Run is a long‑running loop that receives target snapshots // and reconciles Target CRs accordingly -func (m *MessageProcessor) Run(ctx context.Context) error { - m.ctx = ctx +func (r *TargetReconciler) Run(ctx context.Context) error { + r.ctx = ctx - logger := log.FromContext(m.ctx). + logger := log.FromContext(r.ctx). WithValues( - "name", m.targetSource.Name, - "namespace", m.targetSource.Namespace, + "name", r.targetSource.Name, + "namespace", r.targetSource.Namespace, ) logger.Info("target reconciler started") - for m.ctx.Err() == nil { + for r.ctx.Err() == nil { select { - case batch, ok := <-m.in: + case batch, ok := <-r.in: if !ok { // Channel closed, pipeline is shutting down logger.Info("input channel closed, stopping target reconciler") return nil } - m.queue = append(m.queue, batch...) + r.queue = append(r.queue, batch...) case <-ctx.Done(): logger.Info("context canceled, stopping target reconciler") return nil } - for len(m.queue) > 0 { + for len(r.queue) > 0 { if ctx.Err() != nil { return nil // why return nil? } - msg := m.queue[0] - m.queue = m.queue[1:] + msg := r.queue[0] + r.queue = r.queue[1:] - if err := m.processMessage(m.ctx, msg, logger); err != nil { + if err := r.processMessage(r.ctx, msg, logger); err != nil { // Returning error lets the supervisor (controller) // tear down and restart the pipeline via reconciliation // Q: when to return an error vs just log and continue? @@ -92,7 +92,7 @@ func (m *MessageProcessor) Run(ctx context.Context) error { return nil } -func (m *MessageProcessor) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { +func (r *TargetReconciler) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { if err := ctx.Err(); err != nil { return err } @@ -107,7 +107,7 @@ func (m *MessageProcessor) processMessage(ctx context.Context, message core.Disc "index", msg.ChunkIndex, "targetCount", len(msg.Targets), ) - return m.processSnapshot(ctx, msg, logger) + return r.processSnapshot(ctx, msg, logger) case core.DiscoveryEvent: // Process individual event-driven update @@ -115,7 +115,7 @@ func (m *MessageProcessor) processMessage(ctx context.Context, message core.Disc "received discovery event", "target", msg.Target.Name, ) - return m.processEvent(ctx, msg, logger) + return r.processEvent(ctx, msg, logger) default: return fmt.Errorf("unknonw discovery message type %T", msg) @@ -123,18 +123,18 @@ func (m *MessageProcessor) processMessage(ctx context.Context, message core.Disc } // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly -func (m *MessageProcessor) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { - if m.activeSnapshot == nil { - m.startNewSnapshot(chunk, logger) +func (r *TargetReconciler) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { + if r.activeSnapshot == nil { + r.startNewSnapshot(chunk, logger) return nil } - snapshot := m.activeSnapshot + snapshot := r.activeSnapshot // Check if a new snapshot arrived if snapshot.snapshotID != chunk.SnapshotID { // If current snapshot is complete apply it first if snapshot.complete { - if err := m.applySnapshot(ctx, snapshot, logger); err != nil { + if err := r.applySnapshot(ctx, snapshot, logger); err != nil { return err } } else { @@ -147,40 +147,40 @@ func (m *MessageProcessor) processSnapshot(ctx context.Context, chunk core.Disco } // Start collecting the new snapshot - m.startNewSnapshot(chunk, logger) + r.startNewSnapshot(chunk, logger) return nil } - return m.collectSnapshot(chunk, logger) + return r.collectSnapshot(chunk, logger) } -func (m *MessageProcessor) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { - m.activeSnapshot = &snapshotBuffer{ +func (r *TargetReconciler) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { + r.activeSnapshot = &snapshotBuffer{ snapshotID: chunk.SnapshotID, totalChunks: chunk.TotalChunks, received: make(map[int][]core.DiscoveredTarget), complete: false, } // Delete buffered events that will be current with new snapshot - m.deferredEvents = nil + r.deferredEvents = nil - m.collectSnapshot(chunk, logger) + r.collectSnapshot(chunk, logger) } -func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { - snapshot := m.activeSnapshot +func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { + snapshot := r.activeSnapshot if chunk.TotalChunks != snapshot.totalChunks { logger.Error(nil, "snapshot totalChunks mismatch", "snapshotID", snapshot.snapshotID) } if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks { logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex) - m.activeSnapshot = nil + r.activeSnapshot = nil return nil } if _, exists := snapshot.received[chunk.ChunkIndex]; exists { logger.Error(nil, "duplicate snapshot chunk", "index", chunk.ChunkIndex) - m.activeSnapshot = nil + r.activeSnapshot = nil return nil } @@ -193,10 +193,10 @@ func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger return nil } -func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { +func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { select { case <-ctx.Done(): - m.activeSnapshot = nil + r.activeSnapshot = nil return nil default: } @@ -205,7 +205,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot for i := 0; i < snapshot.totalChunks; i++ { select { case <-ctx.Done(): - m.activeSnapshot = nil + r.activeSnapshot = nil return nil default: } @@ -213,7 +213,7 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot chunk, ok := snapshot.received[i] if !ok { logger.Error(nil, "missing snapshot chunk", "index", i) - m.activeSnapshot = nil + r.activeSnapshot = nil return nil } allTargets = append(allTargets, chunk...) @@ -229,34 +229,34 @@ func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshot // a.applyTargets // Replay deferred events - for _, event := range m.deferredEvents { + for _, event := range r.deferredEvents { select { case <-ctx.Done(): return nil default: } - if err := m.applyEvent(ctx, event, logger); err != nil { + if err := r.applyEvent(ctx, event, logger); err != nil { return err } } - m.activeSnapshot = nil - m.deferredEvents = nil + r.activeSnapshot = nil + r.deferredEvents = nil return nil } -func (m *MessageProcessor) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (r *TargetReconciler) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { // If snapshot collecting is active defer events - if m.activeSnapshot != nil { - m.deferredEvents = append(m.deferredEvents, event) + if r.activeSnapshot != nil { + r.deferredEvents = append(r.deferredEvents, event) return nil } // Apply events - return m.applyEvent(ctx, event, logger) + return r.applyEvent(ctx, event, logger) } -func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { switch event.Event { case core.EventCreate: logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 84f9a6f..65a4cf9 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -183,7 +183,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName } // Create target reconciler instance - targetReconciler := discovery.NewMessageProcessor( + targetReconciler := discovery.NewTargetReconciler( r.Client, r.Scheme, targetSource, From c268808d67eb8df1d7328c0658b36bd369eda489 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 10:38:23 -0600 Subject: [PATCH 071/110] moved registry.go to discovery --- internal/controller/discovery/{registry => }/registry.go | 2 +- internal/controller/targetsource_controller.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) rename internal/controller/discovery/{registry => }/registry.go (97%) diff --git a/internal/controller/discovery/registry/registry.go b/internal/controller/discovery/registry.go similarity index 97% rename from internal/controller/discovery/registry/registry.go rename to internal/controller/discovery/registry.go index f2630e8..0afa2b2 100644 --- a/internal/controller/discovery/registry/registry.go +++ b/internal/controller/discovery/registry.go @@ -1,4 +1,4 @@ -package registry +package discovery import ( "fmt" diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 65a4cf9..3b62b6d 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -33,7 +33,6 @@ import ( "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/loaders" "github.com/gnmic/operator/internal/controller/discovery/pipeline" - "github.com/gnmic/operator/internal/controller/discovery/registry" "github.com/go-logr/logr" ) @@ -66,7 +65,7 @@ type TargetSourceReconciler struct { BufferSize int ChunkSize int - DiscoveryRegistry *registry.Registry[types.NamespacedName, core.DiscoveryRegistryValue] + DiscoveryRegistry *discovery.Registry[types.NamespacedName, core.DiscoveryRegistryValue] } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete From 02958966b77f80ee3fc1f0e447b98967e54e9c2a Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 10:39:32 -0600 Subject: [PATCH 072/110] moved supervisor to discovery --- .../controller/discovery/{pipeline => }/supervisor.go | 2 +- internal/controller/targetsource_controller.go | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) rename internal/controller/discovery/{pipeline => }/supervisor.go (99%) diff --git a/internal/controller/discovery/pipeline/supervisor.go b/internal/controller/discovery/supervisor.go similarity index 99% rename from internal/controller/discovery/pipeline/supervisor.go rename to internal/controller/discovery/supervisor.go index 042d305..56fa687 100644 --- a/internal/controller/discovery/pipeline/supervisor.go +++ b/internal/controller/discovery/supervisor.go @@ -1,4 +1,4 @@ -package pipeline +package discovery import ( "context" diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 3b62b6d..301e421 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -32,7 +32,6 @@ import ( "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/loaders" - "github.com/gnmic/operator/internal/controller/discovery/pipeline" "github.com/go-logr/logr" ) @@ -171,7 +170,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName loaderConfigured := targetSource.Spec.Provider != nil webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled - supervisor := pipeline.NewSupervisor(context.Background()) + supervisor := discovery.NewSupervisor(context.Background()) targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) if err := r.DiscoveryRegistry.Register(key, core.DiscoveryRegistryValue{ @@ -190,9 +189,9 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName ) // Start target reconciler reconcilerReady := make(chan struct{}) - supervisor.StartSupervisedComponent(pipeline.ComponentSpec{ + supervisor.StartSupervisedComponent(discovery.ComponentSpec{ Name: "target-reconciler", - Policy: pipeline.RestartPolicy{ + Policy: discovery.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, @@ -221,9 +220,9 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName return err } - supervisor.StartSupervisedComponent(pipeline.ComponentSpec{ + supervisor.StartSupervisedComponent(discovery.ComponentSpec{ Name: "loader", - Policy: pipeline.RestartPolicy{ + Policy: discovery.RestartPolicy{ MaxRestarts: pipelineMaxRestarts, Backoff: pipelineBackoff, }, From 4d32c40fb2e319fa2ff77a9c05f576ba6e0dba4d Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 10:40:26 -0600 Subject: [PATCH 073/110] moved factory.go to discovery/loaders.go --- .../controller/discovery/{loaders/factory.go => loaders.go} | 2 +- internal/controller/targetsource_controller.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) rename internal/controller/discovery/{loaders/factory.go => loaders.go} (97%) diff --git a/internal/controller/discovery/loaders/factory.go b/internal/controller/discovery/loaders.go similarity index 97% rename from internal/controller/discovery/loaders/factory.go rename to internal/controller/discovery/loaders.go index 45bf9c1..0d8ddd3 100644 --- a/internal/controller/discovery/loaders/factory.go +++ b/internal/controller/discovery/loaders.go @@ -1,4 +1,4 @@ -package loaders +package discovery import ( "fmt" diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 301e421..9ba2c94 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -31,7 +31,6 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" - "github.com/gnmic/operator/internal/controller/discovery/loaders" "github.com/go-logr/logr" ) @@ -210,7 +209,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName // Create loader instance if loaderConfigured { - loader, err := loaders.NewLoader( + loader, err := discovery.NewLoader( key, targetSource.Spec, core.LoaderConfig{ChunkSize: r.ChunkSize}, From 7671c1a20aa7a48a26cf306c55ef0698c1ec448f Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 10:54:58 -0600 Subject: [PATCH 074/110] moved send.go to loaders package --- .../discovery/loaders/http/loader.go | 3 ++- .../discovery/{core => loaders}/send.go | 22 ++++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) rename internal/controller/discovery/{core => loaders}/send.go (67%) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 09bb7d6..1e5fc37 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -10,6 +10,7 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/gnmic/operator/internal/controller/discovery/loaders" "github.com/google/uuid" ) @@ -66,7 +67,7 @@ func (l *Loader) Start( }, } - if err := core.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil { + if err := loaders.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil { return err } } diff --git a/internal/controller/discovery/core/send.go b/internal/controller/discovery/loaders/send.go similarity index 67% rename from internal/controller/discovery/core/send.go rename to internal/controller/discovery/loaders/send.go index f24b50c..1377432 100644 --- a/internal/controller/discovery/core/send.go +++ b/internal/controller/discovery/loaders/send.go @@ -1,12 +1,14 @@ -package core +package loaders import ( "context" "fmt" + + "github.com/gnmic/operator/internal/controller/discovery/core" ) // sendMessages sends discovery messages over a channel in a context-aware manner -func sendMessages(ctx context.Context, out chan<- []DiscoveryMessage, messages []DiscoveryMessage) error { +func sendMessages(ctx context.Context, out chan<- []core.DiscoveryMessage, messages []core.DiscoveryMessage) error { select { case <-ctx.Done(): return ctx.Err() @@ -30,14 +32,14 @@ func forEachChunk(total, chunkSize int, fn func(start, end int) error) error { } // createDiscoverySnapshots takes a list of discovered targets and returns chunked DiscoverySnapshots -func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chunkSize int) []DiscoverySnapshot { - var snapshots []DiscoverySnapshot +func createDiscoverySnapshots(targets []core.DiscoveredTarget, snapshotID string, chunkSize int) []core.DiscoverySnapshot { + var snapshots []core.DiscoverySnapshot totalTargets := len(targets) totalChunks := (totalTargets + chunkSize - 1) / chunkSize _ = forEachChunk(totalTargets, chunkSize, func(i, end int) error { chunk := targets[i:end] - snapshots = append(snapshots, DiscoverySnapshot{ + snapshots = append(snapshots, core.DiscoverySnapshot{ Targets: chunk, SnapshotID: snapshotID, ChunkIndex: i / chunkSize, @@ -50,7 +52,7 @@ func createDiscoverySnapshots(targets []DiscoveredTarget, snapshotID string, chu } // SendSnapshot sends discovered targets as a snapshot over a channel in chunks -func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets []DiscoveredTarget, snapshotID string, chunkSize int) error { +func SendSnapshot(ctx context.Context, out chan<- []core.DiscoveryMessage, targets []core.DiscoveredTarget, snapshotID string, chunkSize int) error { if len(targets) == 0 { return fmt.Errorf("no targets in Snapshot") } @@ -58,7 +60,7 @@ func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets [] snapshots := createDiscoverySnapshots(targets, snapshotID, chunkSize) for _, snapshot := range snapshots { // Convert DiscoverySnapshot to DiscoveryMessage - messages := make([]DiscoveryMessage, 1) + messages := make([]core.DiscoveryMessage, 1) messages[0] = snapshot if err := sendMessages(ctx, out, messages); err != nil { @@ -69,8 +71,8 @@ func SendSnapshot(ctx context.Context, out chan<- []DiscoveryMessage, targets [] return nil } -func eventsToMessages(events []DiscoveryEvent) []DiscoveryMessage { - message := make([]DiscoveryMessage, len(events)) +func eventsToMessages(events []core.DiscoveryEvent) []core.DiscoveryMessage { + message := make([]core.DiscoveryMessage, len(events)) for i, event := range events { message[i] = event } @@ -78,7 +80,7 @@ func eventsToMessages(events []DiscoveryEvent) []DiscoveryMessage { } // SendEvents sends discovery messages over channel in a context-aware manner -func SendEvents(ctx context.Context, out chan<- []DiscoveryMessage, events []DiscoveryEvent, chunkSize int) error { +func SendEvents(ctx context.Context, out chan<- []core.DiscoveryMessage, events []core.DiscoveryEvent, chunkSize int) error { if len(events) == 0 { return fmt.Errorf("no events to process") } From 5f1e9cbe91d28e837ff7fbfae4029df45f27c001 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 10:55:59 -0600 Subject: [PATCH 075/110] eliminated message.go --- internal/controller/discovery/core/message.go | 4 ---- internal/controller/discovery/core/message_interface.go | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) delete mode 100644 internal/controller/discovery/core/message.go diff --git a/internal/controller/discovery/core/message.go b/internal/controller/discovery/core/message.go deleted file mode 100644 index af4f6c1..0000000 --- a/internal/controller/discovery/core/message.go +++ /dev/null @@ -1,4 +0,0 @@ -package core - -func (DiscoveryEvent) isDiscoveryMessage() {} -func (DiscoverySnapshot) isDiscoveryMessage() {} diff --git a/internal/controller/discovery/core/message_interface.go b/internal/controller/discovery/core/message_interface.go index 07b819e..0836bc6 100644 --- a/internal/controller/discovery/core/message_interface.go +++ b/internal/controller/discovery/core/message_interface.go @@ -3,3 +3,6 @@ package core type DiscoveryMessage interface { isDiscoveryMessage() } + +func (DiscoveryEvent) isDiscoveryMessage() {} +func (DiscoverySnapshot) isDiscoveryMessage() {} From 6d6753731ca36cdafa5a251e164ed1b70eafd3dc Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 10:56:39 -0600 Subject: [PATCH 076/110] moved const.go to discovery.go --- internal/controller/discovery/client.go | 3 +-- internal/controller/discovery/{core => }/const.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) rename internal/controller/discovery/{core => }/const.go (81%) diff --git a/internal/controller/discovery/client.go b/internal/controller/discovery/client.go index 2deb477..cb02161 100644 --- a/internal/controller/discovery/client.go +++ b/internal/controller/discovery/client.go @@ -6,7 +6,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" - "github.com/gnmic/operator/internal/controller/discovery/core" ) func fetchExistingTargets( @@ -22,7 +21,7 @@ func fetchExistingTargets( &targetList, client.InNamespace(ts.Namespace), client.MatchingLabels{ - core.LabelTargetSourceName: ts.Name, + LabelTargetSourceName: ts.Name, }, ) if err != nil { diff --git a/internal/controller/discovery/core/const.go b/internal/controller/discovery/const.go similarity index 81% rename from internal/controller/discovery/core/const.go rename to internal/controller/discovery/const.go index 82a5962..ac7a57f 100644 --- a/internal/controller/discovery/core/const.go +++ b/internal/controller/discovery/const.go @@ -1,4 +1,4 @@ -package core +package discovery const ( // Labels From 391463097c6caab4b89c72de9789efe8b346e8bf Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 11:28:29 -0600 Subject: [PATCH 077/110] renamed core package within targetsource controller --- internal/controller/targetsource_controller.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 9ba2c94..e52b02b 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -30,7 +30,7 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery" - "github.com/gnmic/operator/internal/controller/discovery/core" + discoveryTypes "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/go-logr/logr" ) @@ -63,7 +63,7 @@ type TargetSourceReconciler struct { BufferSize int ChunkSize int - DiscoveryRegistry *discovery.Registry[types.NamespacedName, core.DiscoveryRegistryValue] + DiscoveryRegistry *discovery.Registry[types.NamespacedName, discoveryTypes.DiscoveryRegistryValue] } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete @@ -171,8 +171,8 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName supervisor := discovery.NewSupervisor(context.Background()) - targetChannel := make(chan []core.DiscoveryMessage, r.BufferSize) - if err := r.DiscoveryRegistry.Register(key, core.DiscoveryRegistryValue{ + targetChannel := make(chan []discoveryTypes.DiscoveryMessage, r.BufferSize) + if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{ Channel: targetChannel, WebhookEnabled: webhookActivated, }); err != nil { @@ -212,7 +212,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName loader, err := discovery.NewLoader( key, targetSource.Spec, - core.LoaderConfig{ChunkSize: r.ChunkSize}, + discoveryTypes.LoaderConfig{ChunkSize: r.ChunkSize}, ) if err != nil { supervisor.Stop() From 46a201fc1d9f0dc9cc73825477f789fc3cb3e860 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 11:34:42 -0600 Subject: [PATCH 078/110] changed events to delete / apply --- internal/controller/discovery/core/types.go | 6 ++---- internal/controller/discovery/target_reconciler.go | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 68c9c7e..2c37fc7 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -15,10 +15,8 @@ type EventAction int const ( // EventDelete indicates that a target should be removed EventDelete EventAction = iota - // EventCreate indicates that a target should be created - EventCreate - // EventUpdate indicates that a target should be updated - EventUpdate + // EventApply indicates that a target should be applied (created or updated) + EventApply ) // DiscoveredTarget represents a target discovered from an external source diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go index 4f3711c..86470c6 100644 --- a/internal/controller/discovery/target_reconciler.go +++ b/internal/controller/discovery/target_reconciler.go @@ -258,12 +258,10 @@ func (r *TargetReconciler) processEvent(ctx context.Context, event core.Discover func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { switch event.Event { - case core.EventCreate: - logger.Info("Would create target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) - case core.EventUpdate: - logger.Info("Would update target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) case core.EventDelete: logger.Info("Would delete target", "name", event.Target.Name) + case core.EventApply: + logger.Info("Would apply target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) } return nil } From 7b17f7e77644abff70f5796704e36b10bf03da15 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 11:37:39 -0600 Subject: [PATCH 079/110] moved send.go into separate utils for loaders --- internal/controller/discovery/loaders/http/loader.go | 4 ++-- internal/controller/discovery/loaders/{ => utils}/send.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename internal/controller/discovery/loaders/{ => utils}/send.go (99%) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 1e5fc37..d7d5961 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -10,7 +10,7 @@ import ( gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" - "github.com/gnmic/operator/internal/controller/discovery/loaders" + loaderUtils "github.com/gnmic/operator/internal/controller/discovery/loaders/utils" "github.com/google/uuid" ) @@ -67,7 +67,7 @@ func (l *Loader) Start( }, } - if err := loaders.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil { + if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil { return err } } diff --git a/internal/controller/discovery/loaders/send.go b/internal/controller/discovery/loaders/utils/send.go similarity index 99% rename from internal/controller/discovery/loaders/send.go rename to internal/controller/discovery/loaders/utils/send.go index 1377432..3cfba8d 100644 --- a/internal/controller/discovery/loaders/send.go +++ b/internal/controller/discovery/loaders/utils/send.go @@ -1,4 +1,4 @@ -package loaders +package utils import ( "context" From 4540163d4137a27a291846a5960ecf09844bf5f8 Mon Sep 17 00:00:00 2001 From: Valentino Diller Date: Wed, 29 Apr 2026 11:45:43 -0600 Subject: [PATCH 080/110] replaced legacy registry package --- cmd/main.go | 4 ++-- internal/apiserver/apiserver.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 4cf6e94..aaf398a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -42,7 +42,7 @@ import ( "github.com/gnmic/operator/internal/apiserver" "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery/core" - "github.com/gnmic/operator/internal/controller/discovery/registry" + "github.com/gnmic/operator/internal/controller/discovery" webhookv1alpha1 "github.com/gnmic/operator/internal/webhook/v1alpha1" //+kubebuilder:scaffold:imports ) @@ -86,7 +86,7 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - discoveryRegistry := registry.NewRegistry[types.NamespacedName, core.DiscoveryRegistryValue]() + discoveryRegistry := discovery.NewRegistry[types.NamespacedName, core.DiscoveryRegistryValue]() mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index a7ca16a..705b277 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -6,7 +6,7 @@ import ( "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery/core" - "github.com/gnmic/operator/internal/controller/discovery/registry" + "github.com/gnmic/operator/internal/controller/discovery" "k8s.io/apimachinery/pkg/types" ) @@ -14,7 +14,7 @@ type APIServer struct { Server *http.Server clusterReconciler *controller.ClusterReconciler - DiscoveryRegistry *registry.Registry[types.NamespacedName, core.DiscoveryRegistryValue] + DiscoveryRegistry *discovery.Registry[types.NamespacedName, core.DiscoveryRegistryValue] } func New(addr string, clusterReconciler *controller.ClusterReconciler) *APIServer { From c728fa2f340066c1f261769ab379ba223e12d62c Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 08:21:11 +0000 Subject: [PATCH 081/110] add supervisor restart policy to targetsource spec configuration --- api/v1alpha1/targetsource_types.go | 13 ++++- api/v1alpha1/zz_generated.deepcopy.go | 30 ++++++++++++ .../operator.gnmic.dev_targetsources.yaml | 7 +++ internal/controller/discovery/defaults.go | 12 +++++ .../controller/targetsource_controller.go | 49 ++++++++++++------- 5 files changed, 92 insertions(+), 19 deletions(-) create mode 100644 internal/controller/discovery/defaults.go diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index a936e66..7c8f74c 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -26,9 +26,12 @@ type TargetSourceSpec struct { Provider *ProviderSpec `json:"provider"` // +kubebuilder:validation:Optional Webhook WebhookSpec `json:"webhook,omitempty"` - // + // +kubebuilder:validation:Optional TargetLabels map[string]string `json:"targetLabels,omitempty"` + // +kubebuilder:validation:Optional + RestartPolicy *RestartPolicySpec `json:"restartPolicy,omitempty"` + // +kubebuilder:validation:MinLength=1 TargetProfile string `json:"targetProfile"` } @@ -54,6 +57,14 @@ type ConsulConfig struct { URL string `json:"url,omitempty"` } +type RestartPolicySpec struct { + // +kubebuilder:validation:Optional + MaxRestarts *int `json:"maxRestarts,omitempty"` + + // +kubebuilder:validation:Optional + BackoffSeconds *int `json:"backoffSeconds,omitempty"` +} + // TargetSourceStatus defines the observed state of TargetSource type TargetSourceStatus struct { Status string `json:"status"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 608d47e..df08573 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -843,6 +843,31 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RestartPolicySpec) DeepCopyInto(out *RestartPolicySpec) { + *out = *in + if in.MaxRestarts != nil { + in, out := &in.MaxRestarts, &out.MaxRestarts + *out = new(int) + **out = **in + } + if in.BackoffSeconds != nil { + in, out := &in.BackoffSeconds, &out.BackoffSeconds + *out = new(int) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestartPolicySpec. +func (in *RestartPolicySpec) DeepCopy() *RestartPolicySpec { + if in == nil { + return nil + } + out := new(RestartPolicySpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceConfig) DeepCopyInto(out *ServiceConfig) { *out = *in @@ -1300,6 +1325,11 @@ func (in *TargetSourceSpec) DeepCopyInto(out *TargetSourceSpec) { (*out)[key] = val } } + if in.RestartPolicy != nil { + in, out := &in.RestartPolicy, &out.RestartPolicy + *out = new(RestartPolicySpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetSourceSpec. diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index b385d8e..6464ea2 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -60,6 +60,13 @@ spec: - message: exactly one of the fields in [http consul] must be set rule: '[has(self.http),has(self.consul)].filter(x,x==true).size() == 1' + restartPolicy: + properties: + backoffSeconds: + type: integer + maxRestarts: + type: integer + type: object targetLabels: additionalProperties: type: string diff --git a/internal/controller/discovery/defaults.go b/internal/controller/discovery/defaults.go new file mode 100644 index 0000000..dc6f046 --- /dev/null +++ b/internal/controller/discovery/defaults.go @@ -0,0 +1,12 @@ +package discovery + +import "time" + +// DefaultRestartPolicy defines the default restart behavior +// for the discovery components +func DefaultRestartPolicy() RestartPolicy { + return RestartPolicy{ + MaxRestarts: 5, + Backoff: 3 * time.Second, + } +} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 06b4fac..fddebda 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -35,11 +35,6 @@ import ( "github.com/go-logr/logr" ) -const ( - pipelineMaxRestarts = 5 - pipelineBackoff = 3 * time.Second -) - // pipelineHandle represents a controller-owned handle to a running pipeline // The controller never manipulates internals; it only invokes cancel() type pipelineHandle struct { @@ -158,6 +153,29 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour return nil } +// resolveRestartPolicy merges an optional spec override with the controller’s default restart policy +func resolveRestartPolicy( + override *gnmicv1alpha1.RestartPolicySpec, +) discovery.RestartPolicy { + defaults := discovery.DefaultRestartPolicy() + + if override == nil { + return defaults + } + + resolved := defaults + + if override.MaxRestarts != nil { + resolved.MaxRestarts = *override.MaxRestarts + } + + if override.BackoffSeconds != nil { + resolved.Backoff = time.Duration(*override.BackoffSeconds) * time.Second + } + + return resolved +} + // startDiscoveryPipeline creates and starts a discovery pipeline for a TargetSource // // Pipeline semantics: @@ -168,6 +186,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error { loaderConfigured := targetSource.Spec.Provider != nil webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled + restartPolicy := resolveRestartPolicy(targetSource.Spec.RestartPolicy) supervisor := discovery.NewSupervisor(context.Background()) @@ -187,22 +206,19 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName targetChannel, ) // Start target reconciler - reconcilerReady := make(chan struct{}) + targetReconcilerReady := make(chan struct{}) supervisor.StartSupervisedComponent(discovery.ComponentSpec{ - Name: "target-reconciler", - Policy: discovery.RestartPolicy{ - MaxRestarts: pipelineMaxRestarts, - Backoff: pipelineBackoff, - }, + Name: "target-reconciler", + Policy: restartPolicy, EscalatesOnFailure: true, Run: func(ctx context.Context) error { - close(reconcilerReady) // Signals that reconciler started successfully + close(targetReconcilerReady) // Signals that reconciler started successfully return targetReconciler.Run(ctx) }, }) // Wait for reconciler to be ready before starting loader select { - case <-reconcilerReady: + case <-targetReconcilerReady: case <-supervisor.Done(): return nil } @@ -220,11 +236,8 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName } supervisor.StartSupervisedComponent(discovery.ComponentSpec{ - Name: "loader", - Policy: discovery.RestartPolicy{ - MaxRestarts: pipelineMaxRestarts, - Backoff: pipelineBackoff, - }, + Name: "loader", + Policy: restartPolicy, EscalatesOnFailure: !webhookActivated, Run: func(ctx context.Context) error { return loader.Start(ctx, key, targetSource.Spec, targetChannel) From 0ff9bdfac9de6e41f4203fc46c4b1ebf54308ce9 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 30 Apr 2026 08:29:31 +0000 Subject: [PATCH 082/110] tags wrapped in labels --- internal/apiserver/apiserver.go | 16 ++++++++++++++-- internal/apiserver/gen.go | 24 +++++++++++++++--------- internal/apiserver/openapi.yaml | 13 ++++++++++--- internal/apiserver/temp.md | 8 ++++++-- 4 files changed, 45 insertions(+), 16 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index b6f04e8..34c09d5 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -118,16 +118,28 @@ func (a *APIServer) CreateTargets(c *gin.Context) { logger.Warn("Received invalid Operation flag") } + labelToMap := make(map[string]string) + if target.Labels != nil { + for _, tag := range *target.Labels { + if tag.Key == nil || tag.Value == nil || *tag.Key == "" { + continue + } + labelToMap[*tag.Key] = *tag.Value + } + } + targets = append(targets, core.DiscoveryEvent{ Target: core.DiscoveredTarget{ Name: target.Name, Address: target.Address, - Labels: map[string]string{"key": "Is this a tag?"}, + Labels: labelToMap, }, Event: event, }) } } + + fmt.Printf("core.DiscoveryEvent was created: %v", targets) core.SendEvents(context.Background(), ch, targets, a.chunkSize) c.JSON(http.StatusOK, payloadTargets) } @@ -140,4 +152,4 @@ func parseURI(c *gin.Context) (url urlStruct) { return } return u -} \ No newline at end of file +} diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index 11e23a2..07a82c2 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -37,13 +37,19 @@ func (e TargetOperation) Valid() bool { } } +// Tag defines model for Tag. +type Tag struct { + Key *string `json:"key,omitempty"` + Value *string `json:"value,omitempty"` +} + // Target defines model for Target. type Target struct { Address string `json:"address"` + Labels *[]Tag `json:"labels,omitempty"` Name string `json:"name"` Operation TargetOperation `json:"operation"` Profile *string `json:"profile,omitempty"` - Tags *[]string `json:"tags,omitempty"` } // TargetOperation defines model for Target.Operation. @@ -135,14 +141,14 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/4yTzYrbMBDHX0VMezSxd3vzrSxlCbTbpdlb6UGVJokWW1JHo0JY/O5FkpO1Ywd60iDN", - "x39+M3oD5XrvLFoO0L5BUEfsZTZfJB2Qk+XJeSQ2mO+l1oQhm3zyCC0EJmMPMFRgZY+rDymBZONsekUb", - "e2h/giKUjBoqiF6PlsYOk/WrWibx5PamWy/A8pAlGcZ+Xdt4IYnkCYahAsI/0RDqJCULry69TQW/K3G/", - "X1FxylXYhCWc8vDVBJ6J+Ui4hxY+1O+06xF1PXJeKDxX2blICp9ukeUrp52Xas3zquH1sJWa1bSnJYuU", - "19i9yxUNp+nA4enbVonvmaAj8ePL7kV8ft5CBX+RQl4CaDbN5m7cDCu9gRY+be43DVTgJR8ztbosyBS2", - "K2Av09lqaOFh5pb6DN7ZUCZy39ylQznLaHOw9L4zKofXr6GsZBnGrXn+5wzDgvL5fp2bxqDI+PItzr5i", - "/BUiRKUwhH3surKvIfa9pNOlYcFjhLGCjyjm2HNI7TuZ+xt/8pzbI/JDFwMjPSe3BbgmHXORE39ByJEs", - "6itxj8hCFTeRyw/DMPwLAAD//2Qu5BBoBAAA", + "H4sIAAAAAAAC/5STz2vbMBTH/xXxtqOJ3e7m2yijFLauLLmNHlT5JVEnS9rTU8EU/+9DkpMmtQPdSc96", + "vz9f+RWU672zaDlA+wpB7bGX2dzIXTo8OY/EGvPlHxzSwYNHaCEwabuDsYIXaSIueMbqcOOenlFxit1I", + "2iHPa8uuIwxhsb6RT2iySzP22fhMuIUWPtVvG9TT+HWa/a21JJJD+rayx8XyaQzJ2tnkRRt7aH+DIpSM", + "HVQQfTdZHRpM1mM1L+LJbbW5QIHwb9SEXSqcx6iO+562f7zIK8yBFcd3Hfg/wGT2C2yKZ+0iKby/xInf", + "Ba29VB9YeDltoWd1utOcRaqr7dbljpoTa9jd/7hT4mcm6Ej8+rbeiK8Pd1DBC1LIkkKzalZXk85Weg0t", + "fFldrxqowEveZ2p1kfsUtitgj+rcddDCzVlY2jN4Z0NR5Lq5SodyltHmZOm90Sqn18+hPLAixiU9P6hh", + "mFE+3C9z6zAo0r488kOsmN64CFEpDGEbjRly4RD7XtJwXFjwlKGt4D2Kc+w5pfZG5v2mv/uc2y3yjYmB", + "kR5S2Axck47zIU/iBSFHsti9G+4WWagSJnL7cRzHfwEAAP//JDP+7tUEAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index 5accf2b..093c1a4 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -56,13 +56,20 @@ components: type: string # in the format "IP:port" profile: type: string - tags: + labels: type: array items: - type: string + $ref: '#/components/schemas/Tag' operation: type: string enum: - created - updated - - deleted \ No newline at end of file + - deleted + Tag: + type: object + properties: + key: + type: string + value: + type: string \ No newline at end of file diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index 715ef09..c70f463 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -1,5 +1,5 @@ ## CURL request -curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"webhook-test", "TargetSourceNameSpace":"default", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' +curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"webhook-test", "TargetSourceNameSpace":"default", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"created","Profile":"defaultProfile", "Labels": [{"key": "tags", "value": "tag1, tag2"}]}]}' ## Empty TargetList curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' @@ -23,8 +23,12 @@ curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" "name": "{{ data.name }}", "address": "{{ data.primary_ip4.address.split('/')[0] if data.primary_ip4 else '' }}:{{ data.custom_fields.port }}", "profile": "{{ data.custom_fields.profile | default('') }}", - "tags": [{% for tag in data.tags %}"{{ tag.name }}"{% if not loop.last %}, {% endif %}{% endfor %}], + "labels": [{ + "key": "tags", + "value": "{{ data.tags | map(attribute='name') | join(', ') }}" + }, "operation": "{{ event }}" + ] } ] } \ No newline at end of file From 589bc9f8cf0643af82f40c4e126ec2e72fc7e67e Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 08:31:37 +0000 Subject: [PATCH 083/110] add targetsource example for lab --- lab/dev/resources/targetsources/cts1.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 lab/dev/resources/targetsources/cts1.yml diff --git a/lab/dev/resources/targetsources/cts1.yml b/lab/dev/resources/targetsources/cts1.yml new file mode 100644 index 0000000..682930c --- /dev/null +++ b/lab/dev/resources/targetsources/cts1.yml @@ -0,0 +1,18 @@ +apiVersion: operator.gnmic.dev/v1alpha1 +kind: TargetSource +metadata: + name: http-discovery +spec: + provider: + http: + url: http://srbsci-121:8081/api/dcim/devices/?export=test + webhook: + enabled: true + targetLabels: + source: inventory + site: siteA + tags: "inventory,siteA,http-discovery" + restartPolicy: + maxRestarts: 2 + backoffSeconds: 4 + targetProfile: eos \ No newline at end of file From a5dde06e8df6dafe7a72f5650e35584ef22b2662 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 08:37:04 +0000 Subject: [PATCH 084/110] remove targetsource example to not add unnecassary logging to main --- lab/dev/resources/targetsources/cts1.yml | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 lab/dev/resources/targetsources/cts1.yml diff --git a/lab/dev/resources/targetsources/cts1.yml b/lab/dev/resources/targetsources/cts1.yml deleted file mode 100644 index 682930c..0000000 --- a/lab/dev/resources/targetsources/cts1.yml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: operator.gnmic.dev/v1alpha1 -kind: TargetSource -metadata: - name: http-discovery -spec: - provider: - http: - url: http://srbsci-121:8081/api/dcim/devices/?export=test - webhook: - enabled: true - targetLabels: - source: inventory - site: siteA - tags: "inventory,siteA,http-discovery" - restartPolicy: - maxRestarts: 2 - backoffSeconds: 4 - targetProfile: eos \ No newline at end of file From 4be9c27a8a547ded1d79f9d6a542da2ad148fe2b Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 08:42:06 +0000 Subject: [PATCH 085/110] update gitignore to not push targetsources in order to prevent logging in main branch --- .gitignore | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 29d31af..7515fa3 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,9 @@ notes/ docs/public docs/resources/_gen/ docs/.hugo_build.lock -test/integration/clab-* \ No newline at end of file +test/integration/clab-* + +# Only for development and testing purposes +# To be removed after development of targetsource +# ignored in order to not add unnecassary logging messages +lab/dev/resources/targetsources \ No newline at end of file From 7962d775ad2e71dece21147e9b9b4dba25104b19 Mon Sep 17 00:00:00 2001 From: Janooski Date: Thu, 30 Apr 2026 09:36:46 +0000 Subject: [PATCH 086/110] refactor --- internal/apiserver/apiserver.go | 100 +++++++++++++++++--------------- 1 file changed, 52 insertions(+), 48 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 34c09d5..8c01486 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -6,7 +6,6 @@ package apiserver import ( "context" - "fmt" "net/http" "github.com/bytedance/gopkg/util/logger" @@ -62,42 +61,33 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { // CreateTargets binds payload to payloadTargets struct defined in openapi contract. Creates a []core.DiscoveryEvent sends it to the core package. func (a *APIServer) CreateTargets(c *gin.Context) { logger.Info("received POST request for CreateTargets.") + var payloadTargets Targets if err := c.ShouldBind(&payloadTargets); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - // the openapi.yaml contract has required fields, but these are not enforced... To enforce them, a middleware - // needs to be used: https://deepwiki.com/oapi-codegen/oapi-codegen/7-middleware-and-validation - // The one for gin-gonic is not actively maintained, so for v1 I'll do validation manually. To be improved. - if payloadTargets.TargetSourceNameSpace == "" { - logger.Error("POST request does not contain value targetSourceNameSpace.") - c.JSON(http.StatusBadRequest, gin.H{"error": "targetSourceNameSpace is required"}) - return - } - if payloadTargets.TargetSourceName == "" { - logger.Error("POST request does not contain value targetSourceName.") - c.JSON(http.StatusBadRequest, gin.H{"error": "targetSourceName is required"}) - return - } - key := types.NamespacedName{ - Namespace: payloadTargets.TargetSourceNameSpace, - Name: payloadTargets.TargetSourceName, - } - ch, ok := a.DiscoveryRegistry.Get(key) + ch, ok := a.DiscoveryRegistry.Get(getKey(payloadTargets)) if !ok { logger.Error("TargetSource ", payloadTargets.TargetSourceNameSpace, "/", payloadTargets.TargetSourceName, "does not exist.") - c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource does not exist"}) + c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource " + payloadTargets.TargetSourceNameSpace + " / " + payloadTargets.TargetSourceName + " does not exist"}) return } - fmt.Printf("payloadTargets %+v\n", payloadTargets) + targets := createDiscoveryEvent(payloadTargets) + // fmt.Printf("core.DiscoveryEvent was created: %v", targets) + core.SendEvents(context.Background(), ch, targets, a.chunkSize) + c.JSON(http.StatusOK, payloadTargets) +} +// createDiscoveryEvent creates object of type core.DiscoveryEvent +func createDiscoveryEvent(payloadTargets Targets) []core.DiscoveryEvent { targets := []core.DiscoveryEvent{} if len(payloadTargets.TargetList) > 0 { for i, target := range payloadTargets.TargetList { if target.Address == "" || target.Name == "" || target.Operation == "" { + // no REST API return here as not all targets might logger.Warn("Target receieved at index", i, " by pull interface does not contain Address, Name or Operation and is skipped.") break } @@ -106,42 +96,56 @@ func (a *APIServer) CreateTargets(c *gin.Context) { break } - event := core.CREATE - switch target.Operation { - case Created: - event = core.UPDATE - case Updated: - event = core.UPDATE - case Deleted: - event = core.DELETE - default: - logger.Warn("Received invalid Operation flag") - } - - labelToMap := make(map[string]string) - if target.Labels != nil { - for _, tag := range *target.Labels { - if tag.Key == nil || tag.Value == nil || *tag.Key == "" { - continue - } - labelToMap[*tag.Key] = *tag.Value - } - } - targets = append(targets, core.DiscoveryEvent{ Target: core.DiscoveredTarget{ Name: target.Name, Address: target.Address, - Labels: labelToMap, + Labels: convertTargetLabelsToMap(target), }, - Event: event, + Event: getEvent(target), }) } } + return targets +} - fmt.Printf("core.DiscoveryEvent was created: %v", targets) - core.SendEvents(context.Background(), ch, targets, a.chunkSize) - c.JSON(http.StatusOK, payloadTargets) +// getKey returns key for used to identify correct channel in DiscoveryRegistry +func getKey(payloadTargets Targets) types.NamespacedName { + key := types.NamespacedName{ + Namespace: payloadTargets.TargetSourceNameSpace, + Name: payloadTargets.TargetSourceName, + } + return key +} + +// convertTargetLabelsToMap converts target.Labels to map. +func convertTargetLabelsToMap(target Target) map[string]string { + labelToMap := make(map[string]string) + if target.Labels != nil { + for _, tag := range *target.Labels { + if tag.Key == nil || tag.Value == nil || *tag.Key == "" { + continue + } + labelToMap[*tag.Key] = *tag.Value + } + } + return labelToMap +} + +// getEvent converts target.Operation to core.Operation. +func getEvent(target Target) core.EventAction { + event := core.CREATE + switch target.Operation { + case Created: + event = core.UPDATE + case Updated: + event = core.UPDATE + case Deleted: + event = core.DELETE + default: + logger.Warn("Received invalid Operation flag") + } + return event } // parseURI parses URI to urlStruct. From 7337541e70e7bbf0867eb2a1e66a7c6ffacc3799 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 09:56:06 +0000 Subject: [PATCH 087/110] add component info to logging --- internal/controller/discovery/target_reconciler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go index 86470c6..3a9f327 100644 --- a/internal/controller/discovery/target_reconciler.go +++ b/internal/controller/discovery/target_reconciler.go @@ -50,6 +50,7 @@ func (r *TargetReconciler) Run(ctx context.Context) error { logger := log.FromContext(r.ctx). WithValues( + "component", "target reconciler", "name", r.targetSource.Name, "namespace", r.targetSource.Namespace, ) From 3ec3203efa7a6e484902ba17dd4eed9085c52277 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 12:21:19 +0000 Subject: [PATCH 088/110] if context is canceled return with ctx.Err() not a clean exit --- internal/controller/discovery/target_reconciler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go index 3a9f327..2f623c3 100644 --- a/internal/controller/discovery/target_reconciler.go +++ b/internal/controller/discovery/target_reconciler.go @@ -73,7 +73,7 @@ func (r *TargetReconciler) Run(ctx context.Context) error { for len(r.queue) > 0 { if ctx.Err() != nil { - return nil // why return nil? + return ctx.Err() } msg := r.queue[0] From 0eaffdcfc63c32bd6d63e46f3081a12015bd76e4 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 12:34:18 +0000 Subject: [PATCH 089/110] applied kubebuilder best-practise logging --- .../discovery/loaders/http/loader.go | 6 +- internal/controller/discovery/supervisor.go | 22 ++++--- .../controller/discovery/target_reconciler.go | 59 ++++++++++++++----- .../controller/targetsource_controller.go | 31 ++++++++-- 4 files changed, 89 insertions(+), 29 deletions(-) diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index d7d5961..67c61e1 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -39,7 +39,11 @@ func (l *Loader) Start( "targetsource", targetsourceNN, ) - logger.Info("HTTP loader started") + logger.Info( + "HTTP loader started", + "targetsource", targetsourceNN.Name, + "namespace", targetsourceNN.Namespace, + ) // Only for debugging: emit a static snapshot every 30 seconds ticker := time.NewTicker(30 * time.Second) diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go index 56fa687..22ec227 100644 --- a/internal/controller/discovery/supervisor.go +++ b/internal/controller/discovery/supervisor.go @@ -78,7 +78,10 @@ func (s *Supervisor) StartSupervisedComponent(component ComponentSpec) { failures := 0 for { - logger.Info("starting component") + logger.Info( + "Starting supervised component", + "component", component.Name, + ) err := component.Run(s.ctx) if s.ctx.Err() != nil { @@ -87,21 +90,26 @@ func (s *Supervisor) StartSupervisedComponent(component ComponentSpec) { } failures++ - logger.Error(err, - "component failed to run", + logger.Error( + err, + "Supervised component failed", + "component", component.Name, "attempt", failures, - "max", component.Policy.MaxRestarts, + "maxRestarts", component.Policy.MaxRestarts, ) if failures >= component.Policy.MaxRestarts { if component.EscalatesOnFailure { - logger.Error(err, - "component permanently failed; shutting down pipeline", + logger.Error( + err, + "Supervised component permanently failed; stopped discovery pipeline", + "component", component.Name, ) s.Stop() } else { logger.Info( - "optional component permanently failed; continuing without it", + "Optional component permanently failed; continuing without it", + "component", component.Name, ) } return diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go index 2f623c3..67d9611 100644 --- a/internal/controller/discovery/target_reconciler.go +++ b/internal/controller/discovery/target_reconciler.go @@ -54,20 +54,30 @@ func (r *TargetReconciler) Run(ctx context.Context) error { "name", r.targetSource.Name, "namespace", r.targetSource.Namespace, ) - logger.Info("target reconciler started") + logger.Info( + "Target reconciler started", + "targetsource", r.targetSource.Name, + "namespace", r.targetSource.Namespace, + ) for r.ctx.Err() == nil { select { case batch, ok := <-r.in: if !ok { // Channel closed, pipeline is shutting down - logger.Info("input channel closed, stopping target reconciler") + logger.Info( + "Input channel closed; stopping target reconciler", + "targetsource", r.targetSource.Name, + ) return nil } r.queue = append(r.queue, batch...) case <-ctx.Done(): - logger.Info("context canceled, stopping target reconciler") + logger.Info( + "Context was canceled; stopping target reconciler", + "targetsource", r.targetSource.Name, + ) return nil } @@ -103,23 +113,24 @@ func (r *TargetReconciler) processMessage(ctx context.Context, message core.Disc case core.DiscoverySnapshot: // Collect snapshot chunks logger.Info( - "received snapshot chunk", + "Received discovery snapshot chunk", "snapshotID", msg.SnapshotID, - "index", msg.ChunkIndex, - "targetCount", len(msg.Targets), + "chunkIndex", msg.ChunkIndex, + "targets", len(msg.Targets), ) return r.processSnapshot(ctx, msg, logger) case core.DiscoveryEvent: // Process individual event-driven update logger.Info( - "received discovery event", + "Received discovery event", + "event", msg.Event, "target", msg.Target.Name, ) return r.processEvent(ctx, msg, logger) default: - return fmt.Errorf("unknonw discovery message type %T", msg) + return fmt.Errorf("Unknown discovery message type %T", msg) } } @@ -142,7 +153,7 @@ func (r *TargetReconciler) processSnapshot(ctx context.Context, chunk core.Disco // If a new snapshot is started before the old one completed // the old one can be discarded logger.Info( - "discarding incomplete snapshot", + "Discarded incomplete discovery snapshot", "snapshotID", snapshot.snapshotID, ) } @@ -172,7 +183,11 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger snapshot := r.activeSnapshot if chunk.TotalChunks != snapshot.totalChunks { - logger.Error(nil, "snapshot totalChunks mismatch", "snapshotID", snapshot.snapshotID) + logger.Error( + nil, + "Snapshot totalChunks mismatch", + "snapshotID", snapshot.snapshotID, + ) } if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks { logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex) @@ -180,7 +195,11 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger return nil } if _, exists := snapshot.received[chunk.ChunkIndex]; exists { - logger.Error(nil, "duplicate snapshot chunk", "index", chunk.ChunkIndex) + logger.Error( + nil, + "Duplicate snapshot chunk received", + "chunkIndex", chunk.ChunkIndex, + ) r.activeSnapshot = nil return nil } @@ -221,9 +240,9 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot } logger.Info( - "applying snapshot", + "Applying discovery snapshot", "snapshotID", snapshot.snapshotID, - "targetCount", len(allTargets), + "targets", len(allTargets), ) // apply all targets @@ -260,9 +279,19 @@ func (r *TargetReconciler) processEvent(ctx context.Context, event core.Discover func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { switch event.Event { case core.EventDelete: - logger.Info("Would delete target", "name", event.Target.Name) + logger.Info( + "Deleting Target", + "target", event.Target.Name, + "targetsource", r.targetSource.Name, + ) case core.EventApply: - logger.Info("Would apply target", "name", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels) + logger.Info( + "Applying Target", + "target", event.Target.Name, + "address", event.Target.Address, + "labels", event.Target.Labels, + "targetsource", r.targetSource.Name, + ) } return nil } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index fddebda..c82ad08 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -71,13 +71,20 @@ type TargetSourceReconciler struct { // move the current state of the cluster closer to the desired state. func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx). - WithName("targetsource controller"). - WithValues("targetsource", req.NamespacedName) + WithName("targetsource-controller"). + WithValues( + "targetsource", req.NamespacedName.Name, + "namespace", req.NamespacedName.Namespace, + ) targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName) // If the TargetSource no longer exists, ensure runtime cleanup if apierrors.IsNotFound(err) { - logger.Info("TargetSource not found; stopping discovery pipeline") + logger.Info( + "TargetSource not found; stopped discovery pipeline", + "targetsource", req.NamespacedName.Name, + "namespace", req.NamespacedName.Namespace, + ) r.stopDiscoveryPipeline(req.NamespacedName) return ctrl.Result{}, nil } else if err != nil { @@ -100,7 +107,11 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } - logger.Info("Discovery pipeline started") + logger.Info( + "Started discovery pipeline", + "targetsource", req.NamespacedName.Name, + "namespace", req.NamespacedName.Namespace, + ) return ctrl.Result{}, nil } @@ -124,7 +135,11 @@ func (r *TargetSourceReconciler) hasPipelineRunning(key types.NamespacedName) bo // reconcileDeletion stops the discovery pipeline and removes the finalizer func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { logger := log.FromContext(ctx) - logger.Info("TargetSource is being deleted, stopping pipeline", "name", key) + logger.Info( + "TargetSource was marked for deletion; stopping discovery pipeline", + "targetsource", key.Name, + "namespace", key.Namespace, + ) r.stopDiscoveryPipeline(key) @@ -250,10 +265,14 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName <-supervisor.Done() supervisor.Wait() // Wait for components to exit - logger.Info("Pipeline stopped; cleaning up") close(targetChannel) r.DiscoveryRegistry.Unregister(key) r.stopDiscoveryPipeline(key) + logger.Info( + "Discovery pipeline stopped; cleaned up resources", + "targetsource", key.Name, + "namespace", key.Namespace, + ) }() r.mu.Lock() From fd4abe7f086416c0bd4b52249a03625ac6d72124 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 12:55:58 +0000 Subject: [PATCH 090/110] improved logging --- .../controller/discovery/target_reconciler.go | 34 +++++++++---------- .../controller/targetsource_controller.go | 29 ++++++++-------- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/target_reconciler.go index 67d9611..39382ab 100644 --- a/internal/controller/discovery/target_reconciler.go +++ b/internal/controller/discovery/target_reconciler.go @@ -48,36 +48,26 @@ func NewTargetReconciler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.T func (r *TargetReconciler) Run(ctx context.Context) error { r.ctx = ctx - logger := log.FromContext(r.ctx). - WithValues( - "component", "target reconciler", - "name", r.targetSource.Name, - "namespace", r.targetSource.Namespace, - ) - logger.Info( - "Target reconciler started", + logger := log.FromContext(ctx).WithValues( + "component", "target-reconciler", "targetsource", r.targetSource.Name, "namespace", r.targetSource.Namespace, ) + logger.Info("Target reconciler started") + for r.ctx.Err() == nil { select { case batch, ok := <-r.in: if !ok { // Channel closed, pipeline is shutting down - logger.Info( - "Input channel closed; stopping target reconciler", - "targetsource", r.targetSource.Name, - ) + logger.Info("Input channel closed; stopping target reconciler") return nil } r.queue = append(r.queue, batch...) case <-ctx.Done(): - logger.Info( - "Context was canceled; stopping target reconciler", - "targetsource", r.targetSource.Name, - ) + logger.Info("Context was canceled; stopping target reconciler") return nil } @@ -190,7 +180,11 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger ) } if chunk.ChunkIndex < 0 || chunk.ChunkIndex >= snapshot.totalChunks { - logger.Error(nil, "snapshot chunk index out of range", "index", chunk.ChunkIndex) + logger.Error( + nil, + "Snapshot chunk index out of range", + "chunkIndex", chunk.ChunkIndex, + ) r.activeSnapshot = nil return nil } @@ -232,7 +226,11 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot chunk, ok := snapshot.received[i] if !ok { - logger.Error(nil, "missing snapshot chunk", "index", i) + logger.Error( + nil, + "Missing snapshot chunk", + "chunkIndex", i, + ) r.activeSnapshot = nil return nil } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index c82ad08..f36d47d 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -80,11 +80,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName) // If the TargetSource no longer exists, ensure runtime cleanup if apierrors.IsNotFound(err) { - logger.Info( - "TargetSource not found; stopped discovery pipeline", - "targetsource", req.NamespacedName.Name, - "namespace", req.NamespacedName.Namespace, - ) + logger.Info("TargetSource not found; stopped discovery pipeline") r.stopDiscoveryPipeline(req.NamespacedName) return ctrl.Result{}, nil } else if err != nil { @@ -100,6 +96,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request } if r.hasPipelineRunning(req.NamespacedName) { + logger.Info("Discovery pipeline already running; reconciliation completed") return ctrl.Result{}, nil } @@ -107,11 +104,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } - logger.Info( - "Started discovery pipeline", - "targetsource", req.NamespacedName.Name, - "namespace", req.NamespacedName.Namespace, - ) + logger.Info("Started discovery pipeline") return ctrl.Result{}, nil } @@ -134,13 +127,11 @@ func (r *TargetSourceReconciler) hasPipelineRunning(key types.NamespacedName) bo // reconcileDeletion stops the discovery pipeline and removes the finalizer func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { - logger := log.FromContext(ctx) - logger.Info( - "TargetSource was marked for deletion; stopping discovery pipeline", + logger := log.FromContext(ctx).WithValues( "targetsource", key.Name, "namespace", key.Namespace, ) - + logger.Info("TargetSource was marked for deletion; stopping discovery pipeline") r.stopDiscoveryPipeline(key) // Remove finalizer if exists @@ -149,6 +140,8 @@ func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key type if err := r.Update(ctx, targetSource); err != nil { return ctrl.Result{}, err } + + logger.Info("Removed TargetSource finalizer") } return ctrl.Result{}, nil @@ -165,6 +158,12 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour return err } + log.FromContext(ctx).Info( + "Added TargetSource finalizer", + "targetsource", targetSource.Name, + "namespace", targetSource.Namespace, + ) + return nil } @@ -234,7 +233,9 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName // Wait for reconciler to be ready before starting loader select { case <-targetReconcilerReady: + logger.Info("Target reconciler started") case <-supervisor.Done(): + logger.Info("Supervisor stopped before target reconciler became ready") return nil } From a6bc11447919eac352164b959bcac482c2b0a115 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Thu, 30 Apr 2026 13:23:47 +0000 Subject: [PATCH 091/110] simplified pipeline context handling --- internal/controller/discovery/core/types.go | 12 +++- internal/controller/discovery/registry.go | 8 +++ .../controller/targetsource_controller.go | 68 ++++++------------- 3 files changed, 38 insertions(+), 50 deletions(-) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 2c37fc7..2f89fdf 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -1,8 +1,18 @@ package core +import "context" + +// DiscoveryRegistryValue represents the controller-owned runtime state +// of a discovery pipeline for a single TargetSource type DiscoveryRegistryValue struct { - Channel chan<- []DiscoveryMessage + // Channel is the outbound communication channel used by discovery + // components (loaders, webhooks, etc.) to emit discovery messages + Channel chan<- []DiscoveryMessage + // WebhookEnabled indicates whether webhook-based discovery is enabled + // for this TargetSource WebhookEnabled bool + // Stop cancels the discovery pipeline associated with this registry entry + Stop context.CancelFunc } type LoaderConfig struct { diff --git a/internal/controller/discovery/registry.go b/internal/controller/discovery/registry.go index 0afa2b2..2193665 100644 --- a/internal/controller/discovery/registry.go +++ b/internal/controller/discovery/registry.go @@ -39,3 +39,11 @@ func (r *Registry[K, V]) Get(key K) (V, bool) { r.mu.RUnlock() return value, ok } + +func (r *Registry[K, V]) Exists(key K) bool { + r.mu.RLock() + defer r.mu.RUnlock() + + _, exists := r.m[key] + return exists +} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index f36d47d..dca0570 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -18,7 +18,6 @@ package controller import ( "context" - "sync" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -35,12 +34,6 @@ import ( "github.com/go-logr/logr" ) -// pipelineHandle represents a controller-owned handle to a running pipeline -// The controller never manipulates internals; it only invokes cancel() -type pipelineHandle struct { - cancel context.CancelFunc -} - // TargetSourceReconciler reconciles a TargetSource object // // Responsibilities: @@ -52,14 +45,13 @@ type TargetSourceReconciler struct { client.Client Scheme *runtime.Scheme - mu sync.Mutex - // runningPipelines tracks currently active pipelines by NamespacedName - runningPipelines map[types.NamespacedName]pipelineHandle - BufferSize int ChunkSize int - DiscoveryRegistry *discovery.Registry[types.NamespacedName, discoveryTypes.DiscoveryRegistryValue] + DiscoveryRegistry *discovery.Registry[ + types.NamespacedName, + discoveryTypes.DiscoveryRegistryValue, + ] } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete @@ -80,8 +72,11 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName) // If the TargetSource no longer exists, ensure runtime cleanup if apierrors.IsNotFound(err) { + if pipeline, ok := r.DiscoveryRegistry.Get(req.NamespacedName); ok { + pipeline.Stop() + r.DiscoveryRegistry.Unregister(req.NamespacedName) + } logger.Info("TargetSource not found; stopped discovery pipeline") - r.stopDiscoveryPipeline(req.NamespacedName) return ctrl.Result{}, nil } else if err != nil { return ctrl.Result{}, err @@ -95,7 +90,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } - if r.hasPipelineRunning(req.NamespacedName) { + if r.DiscoveryRegistry.Exists(req.NamespacedName) { logger.Info("Discovery pipeline already running; reconciliation completed") return ctrl.Result{}, nil } @@ -117,14 +112,6 @@ func (r *TargetSourceReconciler) fetchTargetSource(ctx context.Context, key type return &targetSource, nil } -// hasPipelineRunning checks if a discovery pipeline is already running for the given key -func (r *TargetSourceReconciler) hasPipelineRunning(key types.NamespacedName) bool { - r.mu.Lock() - defer r.mu.Unlock() - _, exists := r.runningPipelines[key] - return exists -} - // reconcileDeletion stops the discovery pipeline and removes the finalizer func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { logger := log.FromContext(ctx).WithValues( @@ -132,7 +119,10 @@ func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key type "namespace", key.Namespace, ) logger.Info("TargetSource was marked for deletion; stopping discovery pipeline") - r.stopDiscoveryPipeline(key) + if pipeline, ok := r.DiscoveryRegistry.Get(key); ok { + pipeline.Stop() + r.DiscoveryRegistry.Unregister(key) + } // Remove finalizer if exists if controllerutil.ContainsFinalizer(targetSource, LabelTargetSourceFinalizer) { @@ -197,7 +187,11 @@ func resolveRestartPolicy( // 2. loader is optional and conditional on spec // 3. Permanent failure of required components shuts down the pipeline // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister -func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger) error { +func (r *TargetSourceReconciler) startDiscoveryPipeline( + key types.NamespacedName, + targetSource *gnmicv1alpha1.TargetSource, + logger logr.Logger, +) error { loaderConfigured := targetSource.Spec.Provider != nil webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled restartPolicy := resolveRestartPolicy(targetSource.Spec.RestartPolicy) @@ -208,6 +202,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{ Channel: targetChannel, WebhookEnabled: webhookActivated, + Stop: supervisor.Stop, }); err != nil { return err } @@ -268,7 +263,6 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName close(targetChannel) r.DiscoveryRegistry.Unregister(key) - r.stopDiscoveryPipeline(key) logger.Info( "Discovery pipeline stopped; cleaned up resources", "targetsource", key.Name, @@ -276,35 +270,11 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline(key types.NamespacedName ) }() - r.mu.Lock() - r.runningPipelines[key] = pipelineHandle{ - cancel: func() { - supervisor.Stop() - }, - } - r.mu.Unlock() - return nil } -// stopDiscoveryPipeline stops and removes a running discovery pipeline -func (r *TargetSourceReconciler) stopDiscoveryPipeline(key types.NamespacedName) { - r.mu.Lock() - running, ok := r.runningPipelines[key] - if ok { - delete(r.runningPipelines, key) - } - r.mu.Unlock() - - if ok { - running.cancel() - } -} - // SetupWithManager sets up the controller with the Manager. func (r *TargetSourceReconciler) SetupWithManager(mgr ctrl.Manager) error { - r.runningPipelines = make(map[types.NamespacedName]pipelineHandle) - return ctrl.NewControllerManagedBy(mgr). For(&gnmicv1alpha1.TargetSource{}). Named("targetsource"). From 535ee49438fb9f4a3b45449a8321d6ab92966e42 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 07:06:36 +0000 Subject: [PATCH 092/110] rename target reconciler to message processor --- ...get_reconciler.go => message_processor.go} | 108 +++++++++--------- .../controller/targetsource_controller.go | 2 +- 2 files changed, 55 insertions(+), 55 deletions(-) rename internal/controller/discovery/{target_reconciler.go => message_processor.go} (68%) diff --git a/internal/controller/discovery/target_reconciler.go b/internal/controller/discovery/message_processor.go similarity index 68% rename from internal/controller/discovery/target_reconciler.go rename to internal/controller/discovery/message_processor.go index 39382ab..ed66940 100644 --- a/internal/controller/discovery/target_reconciler.go +++ b/internal/controller/discovery/message_processor.go @@ -20,8 +20,8 @@ type snapshotBuffer struct { complete bool } -// TargetReconciler consumes discovered targets and applies them to Kubernetes -type TargetReconciler struct { +// MessageProcessor consumes discovery messages and applies them to Kubernetes +type MessageProcessor struct { ctx context.Context client client.Client scheme *runtime.Scheme @@ -33,9 +33,9 @@ type TargetReconciler struct { deferredEvents []core.DiscoveryEvent } -// NewTargetReconciler wires a TargetReconciler instance -func NewTargetReconciler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *TargetReconciler { - return &TargetReconciler{ +// NewMessageProcessor wires a MessageProcessor instance +func NewMessageProcessor(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.TargetSource, in <-chan []core.DiscoveryMessage) *MessageProcessor { + return &MessageProcessor{ client: c, scheme: s, targetSource: ts, @@ -45,41 +45,41 @@ func NewTargetReconciler(c client.Client, s *runtime.Scheme, ts *gnmicv1alpha1.T // Run is a long‑running loop that receives target snapshots // and reconciles Target CRs accordingly -func (r *TargetReconciler) Run(ctx context.Context) error { - r.ctx = ctx +func (m *MessageProcessor) Run(ctx context.Context) error { + m.ctx = ctx logger := log.FromContext(ctx).WithValues( - "component", "target-reconciler", - "targetsource", r.targetSource.Name, - "namespace", r.targetSource.Namespace, + "component", "message-processor", + "targetsource", m.targetSource.Name, + "namespace", m.targetSource.Namespace, ) - logger.Info("Target reconciler started") + logger.Info("Message processor started") - for r.ctx.Err() == nil { + for m.ctx.Err() == nil { select { - case batch, ok := <-r.in: + case batch, ok := <-m.in: if !ok { // Channel closed, pipeline is shutting down - logger.Info("Input channel closed; stopping target reconciler") + logger.Info("Input channel closed; stopping message processor") return nil } - r.queue = append(r.queue, batch...) + m.queue = append(m.queue, batch...) case <-ctx.Done(): - logger.Info("Context was canceled; stopping target reconciler") + logger.Info("Context was canceled; stopping message processor") return nil } - for len(r.queue) > 0 { + for len(m.queue) > 0 { if ctx.Err() != nil { return ctx.Err() } - msg := r.queue[0] - r.queue = r.queue[1:] + msg := m.queue[0] + m.queue = m.queue[1:] - if err := r.processMessage(r.ctx, msg, logger); err != nil { + if err := m.processMessage(m.ctx, msg, logger); err != nil { // Returning error lets the supervisor (controller) // tear down and restart the pipeline via reconciliation // Q: when to return an error vs just log and continue? @@ -89,11 +89,11 @@ func (r *TargetReconciler) Run(ctx context.Context) error { } } - logger.Info("target reconciler stopped") + logger.Info("Message processor stopped") return nil } -func (r *TargetReconciler) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { +func (m *MessageProcessor) processMessage(ctx context.Context, message core.DiscoveryMessage, logger logr.Logger) error { if err := ctx.Err(); err != nil { return err } @@ -108,7 +108,7 @@ func (r *TargetReconciler) processMessage(ctx context.Context, message core.Disc "chunkIndex", msg.ChunkIndex, "targets", len(msg.Targets), ) - return r.processSnapshot(ctx, msg, logger) + return m.processSnapshot(ctx, msg, logger) case core.DiscoveryEvent: // Process individual event-driven update @@ -117,7 +117,7 @@ func (r *TargetReconciler) processMessage(ctx context.Context, message core.Disc "event", msg.Event, "target", msg.Target.Name, ) - return r.processEvent(ctx, msg, logger) + return m.processEvent(ctx, msg, logger) default: return fmt.Errorf("Unknown discovery message type %T", msg) @@ -125,18 +125,18 @@ func (r *TargetReconciler) processMessage(ctx context.Context, message core.Disc } // processSnapshot takes a complete snapshot of discovered targets and reconciles Target CRs accordingly -func (r *TargetReconciler) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { - if r.activeSnapshot == nil { - r.startNewSnapshot(chunk, logger) +func (m *MessageProcessor) processSnapshot(ctx context.Context, chunk core.DiscoverySnapshot, logger logr.Logger) error { + if m.activeSnapshot == nil { + m.startNewSnapshot(chunk, logger) return nil } - snapshot := r.activeSnapshot + snapshot := m.activeSnapshot // Check if a new snapshot arrived if snapshot.snapshotID != chunk.SnapshotID { // If current snapshot is complete apply it first if snapshot.complete { - if err := r.applySnapshot(ctx, snapshot, logger); err != nil { + if err := m.applySnapshot(ctx, snapshot, logger); err != nil { return err } } else { @@ -149,28 +149,28 @@ func (r *TargetReconciler) processSnapshot(ctx context.Context, chunk core.Disco } // Start collecting the new snapshot - r.startNewSnapshot(chunk, logger) + m.startNewSnapshot(chunk, logger) return nil } - return r.collectSnapshot(chunk, logger) + return m.collectSnapshot(chunk, logger) } -func (r *TargetReconciler) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { - r.activeSnapshot = &snapshotBuffer{ +func (m *MessageProcessor) startNewSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) { + m.activeSnapshot = &snapshotBuffer{ snapshotID: chunk.SnapshotID, totalChunks: chunk.TotalChunks, received: make(map[int][]core.DiscoveredTarget), complete: false, } // Delete buffered events that will be current with new snapshot - r.deferredEvents = nil + m.deferredEvents = nil - r.collectSnapshot(chunk, logger) + m.collectSnapshot(chunk, logger) } -func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { - snapshot := r.activeSnapshot +func (m *MessageProcessor) collectSnapshot(chunk core.DiscoverySnapshot, logger logr.Logger) error { + snapshot := m.activeSnapshot if chunk.TotalChunks != snapshot.totalChunks { logger.Error( @@ -185,7 +185,7 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger "Snapshot chunk index out of range", "chunkIndex", chunk.ChunkIndex, ) - r.activeSnapshot = nil + m.activeSnapshot = nil return nil } if _, exists := snapshot.received[chunk.ChunkIndex]; exists { @@ -194,7 +194,7 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger "Duplicate snapshot chunk received", "chunkIndex", chunk.ChunkIndex, ) - r.activeSnapshot = nil + m.activeSnapshot = nil return nil } @@ -207,10 +207,10 @@ func (r *TargetReconciler) collectSnapshot(chunk core.DiscoverySnapshot, logger return nil } -func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { +func (m *MessageProcessor) applySnapshot(ctx context.Context, snapshot *snapshotBuffer, logger logr.Logger) error { select { case <-ctx.Done(): - r.activeSnapshot = nil + m.activeSnapshot = nil return nil default: } @@ -219,7 +219,7 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot for i := 0; i < snapshot.totalChunks; i++ { select { case <-ctx.Done(): - r.activeSnapshot = nil + m.activeSnapshot = nil return nil default: } @@ -231,7 +231,7 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot "Missing snapshot chunk", "chunkIndex", i, ) - r.activeSnapshot = nil + m.activeSnapshot = nil return nil } allTargets = append(allTargets, chunk...) @@ -247,40 +247,40 @@ func (r *TargetReconciler) applySnapshot(ctx context.Context, snapshot *snapshot // a.applyTargets // Replay deferred events - for _, event := range r.deferredEvents { + for _, event := range m.deferredEvents { select { case <-ctx.Done(): return nil default: } - if err := r.applyEvent(ctx, event, logger); err != nil { + if err := m.applyEvent(ctx, event, logger); err != nil { return err } } - r.activeSnapshot = nil - r.deferredEvents = nil + m.activeSnapshot = nil + m.deferredEvents = nil return nil } -func (r *TargetReconciler) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (m *MessageProcessor) processEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { // If snapshot collecting is active defer events - if r.activeSnapshot != nil { - r.deferredEvents = append(r.deferredEvents, event) + if m.activeSnapshot != nil { + m.deferredEvents = append(m.deferredEvents, event) return nil } // Apply events - return r.applyEvent(ctx, event, logger) + return m.applyEvent(ctx, event, logger) } -func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { +func (m *MessageProcessor) applyEvent(ctx context.Context, event core.DiscoveryEvent, logger logr.Logger) error { switch event.Event { case core.EventDelete: logger.Info( "Deleting Target", "target", event.Target.Name, - "targetsource", r.targetSource.Name, + "targetsource", m.targetSource.Name, ) case core.EventApply: logger.Info( @@ -288,7 +288,7 @@ func (r *TargetReconciler) applyEvent(ctx context.Context, event core.DiscoveryE "target", event.Target.Name, "address", event.Target.Address, "labels", event.Target.Labels, - "targetsource", r.targetSource.Name, + "targetsource", m.targetSource.Name, ) } return nil diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index dca0570..b1755b0 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -208,7 +208,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline( } // Create target reconciler instance - targetReconciler := discovery.NewTargetReconciler( + targetReconciler := discovery.NewMessageProcessor( r.Client, r.Scheme, targetSource, From c09c68f5c0c8a0f042b3192458bd4a8f1fe671cf Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 07:16:44 +0000 Subject: [PATCH 093/110] rename pipeline to runtime --- .../controller/targetsource_controller.go | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index b1755b0..d4442cc 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -18,7 +18,6 @@ package controller import ( "context" - "time" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -37,9 +36,9 @@ import ( // TargetSourceReconciler reconciles a TargetSource object // // Responsibilities: -// - Ensure at most one pipeline per TargetSource -// - Start pipelines on reconcile -// - Stop pipelines on deletion or NotFound +// - Ensure at most one runtime per TargetSource +// - Start runtimes on reconcile +// - Stop runtimes on deletion or NotFound // - Delegate runtime failure handling to the Supervisor type TargetSourceReconciler struct { client.Client @@ -72,11 +71,11 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request targetSource, err := r.fetchTargetSource(ctx, req.NamespacedName) // If the TargetSource no longer exists, ensure runtime cleanup if apierrors.IsNotFound(err) { - if pipeline, ok := r.DiscoveryRegistry.Get(req.NamespacedName); ok { - pipeline.Stop() + if runtime, ok := r.DiscoveryRegistry.Get(req.NamespacedName); ok { + runtime.Stop() r.DiscoveryRegistry.Unregister(req.NamespacedName) } - logger.Info("TargetSource not found; stopped discovery pipeline") + logger.Info("TargetSource not found; stopped discovery runtime") return ctrl.Result{}, nil } else if err != nil { return ctrl.Result{}, err @@ -91,15 +90,15 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request } if r.DiscoveryRegistry.Exists(req.NamespacedName) { - logger.Info("Discovery pipeline already running; reconciliation completed") + logger.Info("Discovery runtime already running; reconciliation completed") return ctrl.Result{}, nil } - if err := r.startDiscoveryPipeline(req.NamespacedName, targetSource, logger); err != nil { + if err := r.startDiscoveryRuntime(req.NamespacedName, targetSource, logger); err != nil { return ctrl.Result{}, err } - logger.Info("Started discovery pipeline") + logger.Info("Started discovery runtime") return ctrl.Result{}, nil } @@ -112,15 +111,15 @@ func (r *TargetSourceReconciler) fetchTargetSource(ctx context.Context, key type return &targetSource, nil } -// reconcileDeletion stops the discovery pipeline and removes the finalizer +// reconcileDeletion stops the discovery runtime and removes the finalizer func (r *TargetSourceReconciler) reconcileDeletion(ctx context.Context, key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource) (ctrl.Result, error) { logger := log.FromContext(ctx).WithValues( "targetsource", key.Name, "namespace", key.Namespace, ) - logger.Info("TargetSource was marked for deletion; stopping discovery pipeline") - if pipeline, ok := r.DiscoveryRegistry.Get(key); ok { - pipeline.Stop() + logger.Info("TargetSource was marked for deletion; stopping discovery runtime") + if runtime, ok := r.DiscoveryRegistry.Get(key); ok { + runtime.Stop() r.DiscoveryRegistry.Unregister(key) } @@ -157,6 +156,7 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour return nil } + // resolveRestartPolicy merges an optional spec override with the controller’s default restart policy func resolveRestartPolicy( override *gnmicv1alpha1.RestartPolicySpec, @@ -179,24 +179,19 @@ func resolveRestartPolicy( return resolved } - -// startDiscoveryPipeline creates and starts a discovery pipeline for a TargetSource +// startDiscoveryRuntime creates and starts a discovery runtime for a TargetSource // -// Pipeline semantics: +// Runtime semantics: // 1. target reconciler is mandatory and must start first // 2. loader is optional and conditional on spec -// 3. Permanent failure of required components shuts down the pipeline +// 3. Permanent failure of required components shuts down the runtime // 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister -func (r *TargetSourceReconciler) startDiscoveryPipeline( +func (r *TargetSourceReconciler) startDiscoveryRuntime( key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger, ) error { - loaderConfigured := targetSource.Spec.Provider != nil webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled - restartPolicy := resolveRestartPolicy(targetSource.Spec.RestartPolicy) - - supervisor := discovery.NewSupervisor(context.Background()) targetChannel := make(chan []discoveryTypes.DiscoveryMessage, r.BufferSize) if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{ @@ -264,7 +259,7 @@ func (r *TargetSourceReconciler) startDiscoveryPipeline( close(targetChannel) r.DiscoveryRegistry.Unregister(key) logger.Info( - "Discovery pipeline stopped; cleaned up resources", + "Discovery runtime stopped; cleaned up resources", "targetsource", key.Name, "namespace", key.Namespace, ) From e4c01bac6d1abcdec28af44c20a65b6d4af477e0 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 08:00:34 +0000 Subject: [PATCH 094/110] removed supervisor --- api/v1alpha1/targetsource_types.go | 11 -- .../discovery/core/loader_interface.go | 4 +- internal/controller/discovery/core/types.go | 5 +- internal/controller/discovery/defaults.go | 12 -- .../discovery/loaders/http/loader.go | 2 +- internal/controller/discovery/supervisor.go | 125 --------------- .../controller/targetsource_controller.go | 145 +++++++----------- 7 files changed, 59 insertions(+), 245 deletions(-) delete mode 100644 internal/controller/discovery/defaults.go delete mode 100644 internal/controller/discovery/supervisor.go diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index 7c8f74c..ae719c1 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -29,9 +29,6 @@ type TargetSourceSpec struct { // +kubebuilder:validation:Optional TargetLabels map[string]string `json:"targetLabels,omitempty"` - // +kubebuilder:validation:Optional - RestartPolicy *RestartPolicySpec `json:"restartPolicy,omitempty"` - // +kubebuilder:validation:MinLength=1 TargetProfile string `json:"targetProfile"` } @@ -57,14 +54,6 @@ type ConsulConfig struct { URL string `json:"url,omitempty"` } -type RestartPolicySpec struct { - // +kubebuilder:validation:Optional - MaxRestarts *int `json:"maxRestarts,omitempty"` - - // +kubebuilder:validation:Optional - BackoffSeconds *int `json:"backoffSeconds,omitempty"` -} - // TargetSourceStatus defines the observed state of TargetSource type TargetSourceStatus struct { Status string `json:"status"` diff --git a/internal/controller/discovery/core/loader_interface.go b/internal/controller/discovery/core/loader_interface.go index 72f1898..bebd725 100644 --- a/internal/controller/discovery/core/loader_interface.go +++ b/internal/controller/discovery/core/loader_interface.go @@ -13,9 +13,9 @@ type Loader interface { // Name returns the unique loader identifier e.g. "pull" Name() string - // Start begins discovery and pushes target snapshots or events into the out channel + // Run begins discovery and pushes target snapshots or events into the out channel // The loader must stop cleanly when ctx is canceled - Start( + Run( ctx context.Context, targetsourceName types.NamespacedName, spec gnmicv1alpha1.TargetSourceSpec, diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 2f89fdf..94b4e85 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -8,10 +8,7 @@ type DiscoveryRegistryValue struct { // Channel is the outbound communication channel used by discovery // components (loaders, webhooks, etc.) to emit discovery messages Channel chan<- []DiscoveryMessage - // WebhookEnabled indicates whether webhook-based discovery is enabled - // for this TargetSource - WebhookEnabled bool - // Stop cancels the discovery pipeline associated with this registry entry + // Stop cancels the discovery context associated with this registry entry Stop context.CancelFunc } diff --git a/internal/controller/discovery/defaults.go b/internal/controller/discovery/defaults.go deleted file mode 100644 index dc6f046..0000000 --- a/internal/controller/discovery/defaults.go +++ /dev/null @@ -1,12 +0,0 @@ -package discovery - -import "time" - -// DefaultRestartPolicy defines the default restart behavior -// for the discovery components -func DefaultRestartPolicy() RestartPolicy { - return RestartPolicy{ - MaxRestarts: 5, - Backoff: 3 * time.Second, - } -} diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 67c61e1..383e974 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -27,7 +27,7 @@ func (l *Loader) Name() string { return "http" } -func (l *Loader) Start( +func (l *Loader) Run( ctx context.Context, targetsourceNN types.NamespacedName, spec gnmicv1alpha1.TargetSourceSpec, diff --git a/internal/controller/discovery/supervisor.go b/internal/controller/discovery/supervisor.go deleted file mode 100644 index 22ec227..0000000 --- a/internal/controller/discovery/supervisor.go +++ /dev/null @@ -1,125 +0,0 @@ -package discovery - -import ( - "context" - "sync" - "time" - - "sigs.k8s.io/controller-runtime/pkg/log" -) - -// Supervisor coordinates the runtime lifecycle of pipeline components -// -// Guarantees: -// - Each component is restarted independently -// - Permanent failure escalates according to policy -// - Stop() cancels all components -// - Wait() blocks until all goroutines exit -type Supervisor struct { - ctx context.Context - cancel context.CancelFunc - - wg sync.WaitGroup - - mu sync.Mutex - stopped bool -} - -// RestartPolicy defines restart behavior of a component -type RestartPolicy struct { - MaxRestarts int - Backoff time.Duration -} - -// ComponentSpec defines a supervised component -type ComponentSpec struct { - Name string - Run func(ctx context.Context) error - Policy RestartPolicy - // EscalatesOnFailure indicates whether a permanent failure of this component should shut down the entire pipeline - EscalatesOnFailure bool -} - -// NewSupervisor creates a new Supervisor with a cancellable context -func NewSupervisor(parentCtx context.Context) *Supervisor { - ctx, cancel := context.WithCancel(parentCtx) - return &Supervisor{ - ctx: ctx, - cancel: cancel, - } -} - -// Stop signals all supervised components to stop by canceling the context -func (s *Supervisor) Stop() { - s.mu.Lock() - defer s.mu.Unlock() - - if s.stopped { - return - } - s.stopped = true - s.cancel() -} - -// Done returns a channel that is closed when the pipeline is stopped -func (s *Supervisor) Done() <-chan struct{} { return s.ctx.Done() } - -// Wait blocks until all supervised components have exited -func (s *Supervisor) Wait() { s.wg.Wait() } - -// StartSupervisedComponent starts and supervises a component -func (s *Supervisor) StartSupervisedComponent(component ComponentSpec) { - s.wg.Add(1) - - go func() { - defer s.wg.Done() - - logger := log.FromContext(s.ctx).WithValues("component", component.Name) - failures := 0 - - for { - logger.Info( - "Starting supervised component", - "component", component.Name, - ) - err := component.Run(s.ctx) - - if s.ctx.Err() != nil { - logger.Info("component stopped due to pipeline shutdown") - return - } - - failures++ - logger.Error( - err, - "Supervised component failed", - "component", component.Name, - "attempt", failures, - "maxRestarts", component.Policy.MaxRestarts, - ) - - if failures >= component.Policy.MaxRestarts { - if component.EscalatesOnFailure { - logger.Error( - err, - "Supervised component permanently failed; stopped discovery pipeline", - "component", component.Name, - ) - s.Stop() - } else { - logger.Info( - "Optional component permanently failed; continuing without it", - "component", component.Name, - ) - } - return - } - - select { - case <-time.After(component.Policy.Backoff): - case <-s.ctx.Done(): - return - } - } - }() -} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index d4442cc..afc088b 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -36,10 +36,10 @@ import ( // TargetSourceReconciler reconciles a TargetSource object // // Responsibilities: -// - Ensure at most one runtime per TargetSource -// - Start runtimes on reconcile -// - Stop runtimes on deletion or NotFound -// - Delegate runtime failure handling to the Supervisor +// - Ensure at most one discovery runtime per TargetSource +// - Start runtime on reconcile if not already running +// - Restart runtime on reconcile if spec changed +// - Stop runtime on deletion or NotFound type TargetSourceReconciler struct { client.Client Scheme *runtime.Scheme @@ -94,7 +94,7 @@ func (r *TargetSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, nil } - if err := r.startDiscoveryRuntime(req.NamespacedName, targetSource, logger); err != nil { + if err := r.startDiscovery(req.NamespacedName, targetSource, logger); err != nil { return ctrl.Result{}, err } @@ -156,113 +156,78 @@ func (r *TargetSourceReconciler) ensureFinalizer(ctx context.Context, targetSour return nil } - -// resolveRestartPolicy merges an optional spec override with the controller’s default restart policy -func resolveRestartPolicy( - override *gnmicv1alpha1.RestartPolicySpec, -) discovery.RestartPolicy { - defaults := discovery.DefaultRestartPolicy() - - if override == nil { - return defaults - } - - resolved := defaults - - if override.MaxRestarts != nil { - resolved.MaxRestarts = *override.MaxRestarts - } - - if override.BackoffSeconds != nil { - resolved.Backoff = time.Duration(*override.BackoffSeconds) * time.Second - } - - return resolved -} -// startDiscoveryRuntime creates and starts a discovery runtime for a TargetSource +// startDiscovery creates and starts a discovery runtime for a TargetSource // -// Runtime semantics: -// 1. target reconciler is mandatory and must start first -// 2. loader is optional and conditional on spec -// 3. Permanent failure of required components shuts down the runtime -// 4. Shutdown ordering: cancel ctx -> wait for goroutines to exit -> close channel -> unregister -func (r *TargetSourceReconciler) startDiscoveryRuntime( +// Invariant: +// - MessageProcessor and Loader must run for the lifetime of the TargetSource +// - Any unexpected exit is treated as a bug and triggers full shutdown +func (r *TargetSourceReconciler) startDiscovery( key types.NamespacedName, targetSource *gnmicv1alpha1.TargetSource, logger logr.Logger, ) error { - webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled - targetChannel := make(chan []discoveryTypes.DiscoveryMessage, r.BufferSize) + ctx, cancel := context.WithCancel(context.Background()) + + // Register discovery runtime of targetsource if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{ - Channel: targetChannel, - WebhookEnabled: webhookActivated, - Stop: supervisor.Stop, + Channel: targetChannel, + Stop: cancel, }); err != nil { return err } - // Create target reconciler instance - targetReconciler := discovery.NewMessageProcessor( + // Cleanup function to cleanup discovery runtime of targetsource + cleanup := func() { + cancel() + r.DiscoveryRegistry.Unregister(key) + close(targetChannel) + } + + // Start message processor + messageProcessor := discovery.NewMessageProcessor( r.Client, r.Scheme, targetSource, targetChannel, ) - // Start target reconciler - targetReconcilerReady := make(chan struct{}) - supervisor.StartSupervisedComponent(discovery.ComponentSpec{ - Name: "target-reconciler", - Policy: restartPolicy, - EscalatesOnFailure: true, - Run: func(ctx context.Context) error { - close(targetReconcilerReady) // Signals that reconciler started successfully - return targetReconciler.Run(ctx) - }, - }) - // Wait for reconciler to be ready before starting loader - select { - case <-targetReconcilerReady: - logger.Info("Target reconciler started") - case <-supervisor.Done(): - logger.Info("Supervisor stopped before target reconciler became ready") - return nil - } + go func() { + logger.Info("Message processor started") - // Create loader instance - if loaderConfigured { - loader, err := discovery.NewLoader( - key, - targetSource.Spec, - discoveryTypes.LoaderConfig{ChunkSize: r.ChunkSize}, - ) - if err != nil { - supervisor.Stop() - return err + if err := messageProcessor.Run(ctx); err != nil { + logger.Error(err, "Message processor exited unecpectedly") + } else { + logger.Error(nil, "Message processor exited unexpectedly without error") } - supervisor.StartSupervisedComponent(discovery.ComponentSpec{ - Name: "loader", - Policy: restartPolicy, - EscalatesOnFailure: !webhookActivated, - Run: func(ctx context.Context) error { - return loader.Start(ctx, key, targetSource.Spec, targetChannel) - }, - }) - } + // Any exit is considered a bug that should stop the discovery runtime + cleanup() + }() - // Monitor supervisor in a separate goroutine to handle shutdown and cleanup + // Start target loader + // webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled + loaderConfig := discoveryTypes.LoaderConfig{ + ChunkSize: r.ChunkSize, + } + loader, err := discovery.NewLoader( + key, + targetSource.Spec, + loaderConfig, + ) + if err != nil { + logger.Error(err, "Target loader could not be created") + cleanup() + return err + } go func() { - <-supervisor.Done() - supervisor.Wait() // Wait for components to exit + if err := loader.Run(ctx, key, targetSource.Spec, targetChannel); err != nil { + logger.Error(err, "Target loader exited unexpectedly") + } else { + logger.Error(nil, "Target loader exited unexpectedly without error") + } - close(targetChannel) - r.DiscoveryRegistry.Unregister(key) - logger.Info( - "Discovery runtime stopped; cleaned up resources", - "targetsource", key.Name, - "namespace", key.Namespace, - ) + // Any exit is considered a bug that should stop the discovery runtime + cleanup() }() return nil From 77dbd7e12dd19d33a38113d280b522a7fdea1d99 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 08:13:35 +0000 Subject: [PATCH 095/110] tidy loader configuration abstraction --- .../discovery/core/loader_interface.go | 10 +-------- internal/controller/discovery/core/types.go | 11 ++++++++-- internal/controller/discovery/loaders.go | 12 +++++------ .../discovery/loaders/http/loader.go | 21 +++++++------------ .../controller/targetsource_controller.go | 12 +++++------ 5 files changed, 27 insertions(+), 39 deletions(-) diff --git a/internal/controller/discovery/core/loader_interface.go b/internal/controller/discovery/core/loader_interface.go index bebd725..895258a 100644 --- a/internal/controller/discovery/core/loader_interface.go +++ b/internal/controller/discovery/core/loader_interface.go @@ -2,9 +2,6 @@ package core import ( "context" - - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" - "k8s.io/apimachinery/pkg/types" ) // Loader defines a pluggable TargetSource loader interface @@ -15,10 +12,5 @@ type Loader interface { // Run begins discovery and pushes target snapshots or events into the out channel // The loader must stop cleanly when ctx is canceled - Run( - ctx context.Context, - targetsourceName types.NamespacedName, - spec gnmicv1alpha1.TargetSourceSpec, - out chan<- []DiscoveryMessage, - ) error + Run(ctx context.Context, out chan<- []DiscoveryMessage) error } diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 94b4e85..5028972 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -1,6 +1,11 @@ package core -import "context" +import ( + "context" + + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" + "k8s.io/apimachinery/pkg/types" +) // DiscoveryRegistryValue represents the controller-owned runtime state // of a discovery pipeline for a single TargetSource @@ -13,7 +18,9 @@ type DiscoveryRegistryValue struct { } type LoaderConfig struct { - ChunkSize int + TargetsourceNN types.NamespacedName + Spec *gnmicv1alpha1.TargetSourceSpec + ChunkSize int } // EventAction represents the type of a discovery event diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 0d8ddd3..6c3e133 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -3,22 +3,20 @@ package discovery import ( "fmt" - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" http "github.com/gnmic/operator/internal/controller/discovery/loaders/http" - "k8s.io/apimachinery/pkg/types" ) // NewLoader creates a loader by name -func NewLoader(name types.NamespacedName, spec gnmicv1alpha1.TargetSourceSpec, cfg core.LoaderConfig) (core.Loader, error) { +func NewLoader(cfg core.LoaderConfig) (core.Loader, error) { switch { - case spec.Provider.HTTP != nil: + case cfg.Spec.Provider.HTTP != nil: return http.New(cfg), nil - case spec.Provider.Consul != nil: - return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", name) + case cfg.Spec.Provider.Consul != nil: + return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) default: - return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", name) + return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) } } diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 383e974..17812aa 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -5,10 +5,8 @@ import ( "fmt" "time" - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/log" - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" loaderUtils "github.com/gnmic/operator/internal/controller/discovery/loaders/utils" "github.com/google/uuid" @@ -27,22 +25,17 @@ func (l *Loader) Name() string { return "http" } -func (l *Loader) Run( - ctx context.Context, - targetsourceNN types.NamespacedName, - spec gnmicv1alpha1.TargetSourceSpec, - out chan<- []core.DiscoveryMessage, -) error { +func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) error { logger := log.FromContext(ctx).WithValues( "component", "loader", "name", l.Name(), - "targetsource", targetsourceNN, + "targetsource", l.cfg.TargetsourceNN, ) logger.Info( "HTTP loader started", - "targetsource", targetsourceNN.Name, - "namespace", targetsourceNN.Namespace, + "targetsource", l.cfg.TargetsourceNN.Name, + "namespace", l.cfg.TargetsourceNN.Namespace, ) // Only for debugging: emit a static snapshot every 30 seconds @@ -57,17 +50,17 @@ func (l *Loader) Run( case <-ticker.C: // Example snapshot (placeholder) - snapshotID := fmt.Sprintf("snapshot-%s-%s", targetsourceNN, uuid.NewString()) + snapshotID := fmt.Sprintf("%s-%s-%s", l.cfg.TargetsourceNN.Namespace, l.cfg.TargetsourceNN.Name, uuid.NewString()) targets := []core.DiscoveredTarget{ { Name: "ceos1", Address: "clab-3-nodes-ceos1:6030", - Labels: map[string]string{"TargetSource": targetsourceNN.String()}, + Labels: map[string]string{"TargetSource": l.cfg.TargetsourceNN.String()}, }, { Name: "leaf1", Address: "clab-3-nodes-leaf1:57400", - Labels: map[string]string{"TargetSource": targetsourceNN.String()}, + Labels: map[string]string{"TargetSource": l.cfg.TargetsourceNN.String()}, }, } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index afc088b..0064570 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -207,20 +207,18 @@ func (r *TargetSourceReconciler) startDiscovery( // Start target loader // webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled loaderConfig := discoveryTypes.LoaderConfig{ - ChunkSize: r.ChunkSize, + TargetsourceNN: key, + Spec: &targetSource.Spec, + ChunkSize: r.ChunkSize, } - loader, err := discovery.NewLoader( - key, - targetSource.Spec, - loaderConfig, - ) + loader, err := discovery.NewLoader(loaderConfig) if err != nil { logger.Error(err, "Target loader could not be created") cleanup() return err } go func() { - if err := loader.Run(ctx, key, targetSource.Spec, targetChannel); err != nil { + if err := loader.Run(ctx, targetChannel); err != nil { logger.Error(err, "Target loader exited unexpectedly") } else { logger.Error(nil, "Target loader exited unexpectedly without error") From fe900e38774051956054d70af6ee24da88beb71f Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 08:14:40 +0000 Subject: [PATCH 096/110] regenearte manifests without restartPolicy --- api/v1alpha1/zz_generated.deepcopy.go | 30 ------------------- .../operator.gnmic.dev_targetsources.yaml | 7 ----- 2 files changed, 37 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index df08573..608d47e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -843,31 +843,6 @@ func (in *ProviderSpec) DeepCopy() *ProviderSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RestartPolicySpec) DeepCopyInto(out *RestartPolicySpec) { - *out = *in - if in.MaxRestarts != nil { - in, out := &in.MaxRestarts, &out.MaxRestarts - *out = new(int) - **out = **in - } - if in.BackoffSeconds != nil { - in, out := &in.BackoffSeconds, &out.BackoffSeconds - *out = new(int) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestartPolicySpec. -func (in *RestartPolicySpec) DeepCopy() *RestartPolicySpec { - if in == nil { - return nil - } - out := new(RestartPolicySpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceConfig) DeepCopyInto(out *ServiceConfig) { *out = *in @@ -1325,11 +1300,6 @@ func (in *TargetSourceSpec) DeepCopyInto(out *TargetSourceSpec) { (*out)[key] = val } } - if in.RestartPolicy != nil { - in, out := &in.RestartPolicy, &out.RestartPolicy - *out = new(RestartPolicySpec) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetSourceSpec. diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index 6464ea2..b385d8e 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -60,13 +60,6 @@ spec: - message: exactly one of the fields in [http consul] must be set rule: '[has(self.http),has(self.consul)].filter(x,x==true).size() == 1' - restartPolicy: - properties: - backoffSeconds: - type: integer - maxRestarts: - type: integer - type: object targetLabels: additionalProperties: type: string From c1d7a91bc7e79f87736454dc31e530d3cf89b7fd Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 08:29:54 +0000 Subject: [PATCH 097/110] tidy up comments --- internal/controller/discovery/discovery.go | 4 +--- internal/controller/discovery/loaders.go | 1 + internal/controller/targetsource_controller.go | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/controller/discovery/discovery.go b/internal/controller/discovery/discovery.go index 3dc51bd..491cdfb 100644 --- a/internal/controller/discovery/discovery.go +++ b/internal/controller/discovery/discovery.go @@ -4,13 +4,11 @@ package discovery // // The discovery subsystem is responsible for: // - Receiving discovery data from external providers (loaders, webhooks). -// - Supervising discovery pipelines and restart semantics. // - Applying discovered state to Kubernetes Targets. // // The package is structured into the following subpackages: // - core: message contracts, snapshot/event types, and transport helpers. -// - pipeline: supervision, restart policies, and lifecycle control. -// - reconciler: snapshot + event target state application logic. +// - message processor: snapshot + event target state application logic. // - loaders: target discovery providers (HTTP, webhook, etc.). // - registry: key -> channel registry. // diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 6c3e133..9704b16 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -12,6 +12,7 @@ func NewLoader(cfg core.LoaderConfig) (core.Loader, error) { switch { case cfg.Spec.Provider.HTTP != nil: + // webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled return http.New(cfg), nil case cfg.Spec.Provider.Consul != nil: return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 0064570..2ba18a2 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -205,7 +205,6 @@ func (r *TargetSourceReconciler) startDiscovery( }() // Start target loader - // webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled loaderConfig := discoveryTypes.LoaderConfig{ TargetsourceNN: key, Spec: &targetSource.Spec, From 5c37a2a68557bb3a0fa142946f8593edca2be9ac Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 13:14:22 +0000 Subject: [PATCH 098/110] add dynamic endpoints --- api/v1alpha1/zz_generated.deepcopy.go | 5 + cmd/main.go | 20 ++-- internal/apiserver/apiserver.go | 111 +++++------------- internal/apiserver/targets.go | 71 +++++++++++ internal/controller/discovery/core/types.go | 4 +- internal/controller/discovery/loaders.go | 10 +- .../discovery/loaders/utils/endpoint.go | 22 ++++ .../controller/targetsource_controller.go | 4 + 8 files changed, 150 insertions(+), 97 deletions(-) create mode 100644 internal/apiserver/targets.go create mode 100644 internal/controller/discovery/loaders/utils/endpoint.go diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 608d47e..3656535 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -826,6 +826,11 @@ func (in *ProviderSpec) DeepCopyInto(out *ProviderSpec) { *out = new(HTTPConfig) **out = **in } + if in.PULL != nil { + in, out := &in.PULL, &out.PULL + *out = new(WebhookSpec) + (*in).DeepCopyInto(*out) + } if in.Consul != nil { in, out := &in.Consul, &out.Consul *out = new(ConsulConfig) diff --git a/cmd/main.go b/cmd/main.go index e239a88..eadc12d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -43,8 +43,8 @@ import ( operatorv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/apiserver" "github.com/gnmic/operator/internal/controller" - "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery" + "github.com/gnmic/operator/internal/controller/discovery/core" webhookv1alpha1 "github.com/gnmic/operator/internal/webhook/v1alpha1" //+kubebuilder:scaffold:imports ) @@ -127,12 +127,22 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "Pipeline") os.Exit(1) } + + var api *apiserver.APIServer + if apiAddr != "" { + api, err = apiserver.New(apiAddr, clusterReconciler, discoveryRegistry, discoveryChunkSize) + if err != nil { + setupLog.Error(err, "unable to initialize API server") + os.Exit(1) + } + } if err := (&controller.TargetSourceReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), BufferSize: discoveryBufferSize, ChunkSize: discoveryChunkSize, DiscoveryRegistry: discoveryRegistry, + APIRouter: api.Router(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TargetSource") os.Exit(1) @@ -232,13 +242,7 @@ func main() { os.Exit(1) } - if apiAddr != "" { - api, err := apiserver.New(apiAddr, clusterReconciler, discoveryChunkSize) - if err != nil { - setupLog.Error(err, "unable to intialize gin API server") - os.Exit(1) - } - api.DiscoveryRegistry = discoveryRegistry + if api != nil { err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { errCh := make(chan error) go func() { diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 9e9131a..4593737 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -5,14 +5,12 @@ package apiserver // Then use: go generate ./internal/apiserver import ( - "context" "net/http" - "github.com/bytedance/gopkg/util/logger" "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" - "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery" + "github.com/gnmic/operator/internal/controller/discovery/core" "k8s.io/apimachinery/pkg/types" ) @@ -20,16 +18,27 @@ type APIServer struct { Server *http.Server router *gin.Engine clusterReconciler *controller.ClusterReconciler - - DiscoveryRegistry *discovery.Registry[types.NamespacedName, core.DiscoveryRegistryValue] + DiscoveryRegistry *discovery.Registry[ + types.NamespacedName, + core.DiscoveryRegistryValue, + ] + ChunkSize int } type urlStruct struct { - namespace string `uri:"namespace" binding:"required"` + namespace string `uri:"namespace" binding:"required"` gNMIcClusterName string `uri:"gNMIcClusterName" binding:"required"` } -func New(addr string, clusterReconciler *controller.ClusterReconciler, chunkSize int) (*APIServer, error) { +func New( + addr string, + clusterReconciler *controller.ClusterReconciler, + discoveryRegistry *discovery.Registry[ + types.NamespacedName, + core.DiscoveryRegistryValue, + ], + discoveryChunksize int, +) (*APIServer, error) { router := gin.Default() a := &APIServer{ Server: &http.Server{ @@ -38,15 +47,26 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler, chunkSize }, router: router, clusterReconciler: clusterReconciler, - chunkSize: chunkSize, + DiscoveryRegistry: discoveryRegistry, + ChunkSize: discoveryChunksize, } - apiBaseURL := "/api/v1/:namespace/:gNMIcClusterName" - RegisterHandlersWithOptions(router, a, GinServerOptions{BaseURL: apiBaseURL}) + // apiBaseURL := "/api/v1/:namespace/:gNMIcClusterName" + // RegisterHandlersWithOptions(router, a, GinServerOptions{BaseURL: apiBaseURL}) + a.routes() return a, nil } // kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 +func (a *APIServer) Router() *gin.Engine { + return a.router +} + +func (a *APIServer) routes() { + a.router.GET("/clusters/:namespace/:name/plan", a.GetClusterPlan) + a.router.POST("/api/v1/:namespace/target-source/:name/createTargets", a.CreateTargets) +} + // GetClusterPlan returns cluster plan func (a *APIServer) GetClusterPlan(c *gin.Context) { url := parseURI(c) @@ -58,77 +78,6 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { c.JSON(200, plan) } -// CreateTargets binds payload to payloadTargets struct defined in openapi contract. Creates a []core.DiscoveryEvent sends it to the core package. -func (a *APIServer) CreateTargets(c *gin.Context) { - // Discussion with Daniel: this was input from Jan and Karim that the URI should be a template - // But I don't think it is needed in the CreateTargets function - // url := parseURI(c) - // fmt.Printf("namespace: %s", url.namespace) - // fmt.Printf("gNMIcClusterName: %s", url.gNMIcClusterName) - - var payloadTargets Targets - if err := c.ShouldBind(&payloadTargets); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - - // the openapi.yaml contract has required fields, but these are not enforced... To enforce them, a middleware - // needs to be used: https://deepwiki.com/oapi-codegen/oapi-codegen/7-middleware-and-validation - // The one for gin-gonic is not actively maintained, so for v1 I'll do validation manually. To be improved. - if payloadTargets.TargetSourceNameSpace == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "targetSourceNameSpace is required"}) - return - } - if payloadTargets.TargetSourceName == "" { - c.JSON(http.StatusBadRequest, gin.H{"error": "targetSourceName is required"}) - return - } - - targets := []core.DiscoveryEvent{} - if len(payloadTargets.TargetList) > 0 { - for i, target := range payloadTargets.TargetList { - if target.Address == "" || target.Name == "" || target.Operation == "" { - logger.Warn("Target receieved at index", i , " by pull interface does not contain Address, Name or Operation and is skipped.") - break - } - if target.Operation.Valid() != true { - logger.Warn("Target receieved at index", i , " by pull interface has invalid Operation.") - break - } - - event := core.CREATE - switch target.Operation { - case Create: - event = core.CREATE - case Delete: - event = core.DELETE - } - - targets = append(targets, core.DiscoveryEvent{ - Target: core.DiscoveredTarget{ - Name: target.Name, - Address: target.Address, - Labels: map[string]string{"key": "Is this a tag?"}, - }, - Event: event, - }) - } - } - - key := types.NamespacedName{ - Namespace: payloadTargets.TargetSourceNameSpace, - Name: payloadTargets.TargetSourceName, - } - ch, ok := a.DiscoveryRegistry.Get(key) - if !ok { - logger.Error("TargetSource " , payloadTargets.TargetSourceNameSpace, "/", payloadTargets.TargetSourceName, "does not exist.") - c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource does not exist"}) - return - } - core.SendEvents(context.Background(), ch, targets, a.chunkSize) - c.JSON(http.StatusOK, payloadTargets) -} - // parseURI parses URI to urlStruct. func parseURI(c *gin.Context) (url urlStruct) { var u urlStruct diff --git a/internal/apiserver/targets.go b/internal/apiserver/targets.go new file mode 100644 index 0000000..0e2c48f --- /dev/null +++ b/internal/apiserver/targets.go @@ -0,0 +1,71 @@ +package apiserver + +import ( + "context" + "net/http" + + "github.com/bytedance/gopkg/util/logger" + "github.com/gin-gonic/gin" + "github.com/gnmic/operator/internal/controller/discovery/core" + "github.com/gnmic/operator/internal/controller/discovery/loaders/utils" + "k8s.io/apimachinery/pkg/types" +) + +// CreateTargets binds payload to payloadTargets struct defined in openapi contract. +// Creates a []core.DiscoveryEvent sends it to the core package. +func (a *APIServer) CreateTargets(c *gin.Context) { + namespace := c.Param("namespace") + name := c.Param("name") + key := types.NamespacedName{ + Namespace: namespace, + Name: name, + } + + registry, ok := a.DiscoveryRegistry.Get(key) + if !ok { + c.JSON(http.StatusNotFound, gin.H{ + "error": "TargetSource not active or does not exist", + }) + return + } + + var payloadTargets Targets + if err := c.ShouldBind(&payloadTargets); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + targets := []core.DiscoveryEvent{} + if len(payloadTargets.TargetList) > 0 { + for i, target := range payloadTargets.TargetList { + if target.Address == "" || target.Name == "" || target.Operation == "" { + logger.Warn("Target receieved at index", i, " by pull interface does not contain Address, Name or Operation and is skipped.") + break + } + if target.Operation.Valid() != true { + logger.Warn("Target receieved at index", i, " by pull interface has invalid Operation.") + break + } + + event := core.EventApply + switch target.Operation { + case Create: + event = core.EventApply + case Delete: + event = core.EventDelete + } + + targets = append(targets, core.DiscoveryEvent{ + Target: core.DiscoveredTarget{ + Name: target.Name, + Address: target.Address, + Labels: map[string]string{"key": "Is this a tag?"}, + }, + Event: event, + }) + } + } + + utils.SendEvents(context.Background(), registry.Channel, targets, a.ChunkSize) + c.JSON(http.StatusOK, payloadTargets) +} diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 5028972..12490e9 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -3,12 +3,13 @@ package core import ( "context" + "github.com/gin-gonic/gin" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "k8s.io/apimachinery/pkg/types" ) // DiscoveryRegistryValue represents the controller-owned runtime state -// of a discovery pipeline for a single TargetSource +// and its configuration type DiscoveryRegistryValue struct { // Channel is the outbound communication channel used by discovery // components (loaders, webhooks, etc.) to emit discovery messages @@ -21,6 +22,7 @@ type LoaderConfig struct { TargetsourceNN types.NamespacedName Spec *gnmicv1alpha1.TargetSourceSpec ChunkSize int + Router *gin.Engine } // EventAction represents the type of a discovery event diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 9704b16..02669cb 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -4,20 +4,16 @@ import ( "fmt" "github.com/gnmic/operator/internal/controller/discovery/core" - http "github.com/gnmic/operator/internal/controller/discovery/loaders/http" + "github.com/gnmic/operator/internal/controller/discovery/loaders/http" ) -// NewLoader creates a loader by name func NewLoader(cfg core.LoaderConfig) (core.Loader, error) { - switch { case cfg.Spec.Provider.HTTP != nil: - // webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled return http.New(cfg), nil - case cfg.Spec.Provider.Consul != nil: - return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) + // webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled + default: return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) } - } diff --git a/internal/controller/discovery/loaders/utils/endpoint.go b/internal/controller/discovery/loaders/utils/endpoint.go new file mode 100644 index 0000000..ef83f18 --- /dev/null +++ b/internal/controller/discovery/loaders/utils/endpoint.go @@ -0,0 +1,22 @@ +package utils + +import ( + "fmt" + + "github.com/gin-gonic/gin" + "k8s.io/apimachinery/pkg/types" +) + +func CreateTargetsPath( + router *gin.Engine, + nn types.NamespacedName, + handler gin.HandlerFunc, +) { + path := fmt.Sprintf( + "/api/v1/%s/target-source/%s/createTargets", + nn.Namespace, + nn.Name, + ) + + router.POST(path, handler) +} diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 2ba18a2..935139c 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/gin-gonic/gin" gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery" discoveryTypes "github.com/gnmic/operator/internal/controller/discovery/core" @@ -51,6 +52,8 @@ type TargetSourceReconciler struct { types.NamespacedName, discoveryTypes.DiscoveryRegistryValue, ] + + APIRouter *gin.Engine } // +kubebuilder:rbac:groups=operator.gnmic.dev,resources=targetsources,verbs=get;list;watch;create;update;patch;delete @@ -209,6 +212,7 @@ func (r *TargetSourceReconciler) startDiscovery( TargetsourceNN: key, Spec: &targetSource.Spec, ChunkSize: r.ChunkSize, + Router: r.APIRouter, } loader, err := discovery.NewLoader(loaderConfig) if err != nil { From 05c7538ce47a4b81fd245b11435cb13481c4c671 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 13:23:34 +0000 Subject: [PATCH 099/110] move webhook spec into provider and rename it to acceptPush --- api/v1alpha1/targetsource_types.go | 10 +++------- internal/controller/discovery/core/types.go | 5 ++++- internal/controller/discovery/loaders.go | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/api/v1alpha1/targetsource_types.go b/api/v1alpha1/targetsource_types.go index ae719c1..3d69743 100644 --- a/api/v1alpha1/targetsource_types.go +++ b/api/v1alpha1/targetsource_types.go @@ -24,8 +24,7 @@ import ( // +kubebuilder:validation:Required type TargetSourceSpec struct { Provider *ProviderSpec `json:"provider"` - // +kubebuilder:validation:Optional - Webhook WebhookSpec `json:"webhook,omitempty"` + // +kubebuilder:validation:Optional TargetLabels map[string]string `json:"targetLabels,omitempty"` @@ -39,14 +38,11 @@ type ProviderSpec struct { Consul *ConsulConfig `json:"consul,omitempty"` } -type WebhookSpec struct { - // +kubebuilder:validation:Optional - Enabled *bool `json:"enabled,omitempty"` -} - type HTTPConfig struct { // +kubebuilder:validation:MinLength=1 URL string `json:"url"` + // +kubebuilder:validation:Optional + AcceptPush bool `json:"acceptPush,omitempty"` } type ConsulConfig struct { diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 5028972..1dfcc9f 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -8,19 +8,22 @@ import ( ) // DiscoveryRegistryValue represents the controller-owned runtime state -// of a discovery pipeline for a single TargetSource +// with its configuration for a single TargetSource type DiscoveryRegistryValue struct { // Channel is the outbound communication channel used by discovery // components (loaders, webhooks, etc.) to emit discovery messages Channel chan<- []DiscoveryMessage // Stop cancels the discovery context associated with this registry entry Stop context.CancelFunc + + LoaderConfig *LoaderConfig } type LoaderConfig struct { TargetsourceNN types.NamespacedName Spec *gnmicv1alpha1.TargetSourceSpec ChunkSize int + AcceptPush bool } // EventAction represents the type of a discovery event diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 9704b16..487c76b 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -12,7 +12,7 @@ func NewLoader(cfg core.LoaderConfig) (core.Loader, error) { switch { case cfg.Spec.Provider.HTTP != nil: - // webhookActivated := targetSource.Spec.Webhook.Enabled != nil && *targetSource.Spec.Webhook.Enabled + cfg.AcceptPush = cfg.Spec.Provider.HTTP.AcceptPush return http.New(cfg), nil case cfg.Spec.Provider.Consul != nil: return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) From 061d4b83daac3a5c26fbe12151aa353a88d1a213 Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 13:24:00 +0000 Subject: [PATCH 100/110] regenerate manifests --- api/v1alpha1/zz_generated.deepcopy.go | 21 ------------------- .../operator.gnmic.dev_targetsources.yaml | 7 ++----- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 608d47e..61e81fd 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1292,7 +1292,6 @@ func (in *TargetSourceSpec) DeepCopyInto(out *TargetSourceSpec) { *out = new(ProviderSpec) (*in).DeepCopyInto(*out) } - in.Webhook.DeepCopyInto(&out.Webhook) if in.TargetLabels != nil { in, out := &in.TargetLabels, &out.TargetLabels *out = make(map[string]string, len(*in)) @@ -1478,23 +1477,3 @@ func (in *TunnelTargetPolicyStatus) DeepCopy() *TunnelTargetPolicyStatus { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *WebhookSpec) DeepCopyInto(out *WebhookSpec) { - *out = *in - if in.Enabled != nil { - in, out := &in.Enabled, &out.Enabled - *out = new(bool) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookSpec. -func (in *WebhookSpec) DeepCopy() *WebhookSpec { - if in == nil { - return nil - } - out := new(WebhookSpec) - in.DeepCopyInto(out) - return out -} diff --git a/config/crd/bases/operator.gnmic.dev_targetsources.yaml b/config/crd/bases/operator.gnmic.dev_targetsources.yaml index b385d8e..37d6919 100644 --- a/config/crd/bases/operator.gnmic.dev_targetsources.yaml +++ b/config/crd/bases/operator.gnmic.dev_targetsources.yaml @@ -49,6 +49,8 @@ spec: type: object http: properties: + acceptPush: + type: boolean url: minLength: 1 type: string @@ -67,11 +69,6 @@ spec: targetProfile: minLength: 1 type: string - webhook: - properties: - enabled: - type: boolean - type: object required: - provider - targetProfile From f8b92b2c56872357418ebcc135259f5c5f86e2ef Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 13:56:38 +0000 Subject: [PATCH 101/110] verify acceptPush is enabled for webhook config --- internal/apiserver/targets.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/apiserver/targets.go b/internal/apiserver/targets.go index 0e2c48f..2e434b9 100644 --- a/internal/apiserver/targets.go +++ b/internal/apiserver/targets.go @@ -22,7 +22,7 @@ func (a *APIServer) CreateTargets(c *gin.Context) { } registry, ok := a.DiscoveryRegistry.Get(key) - if !ok { + if !ok || (registry.LoaderConfig.AcceptPush != true) { c.JSON(http.StatusNotFound, gin.H{ "error": "TargetSource not active or does not exist", }) From 41655a0d4e835bc2ff0b8a5a1cdaf55aa4bdfd7a Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 14:12:05 +0000 Subject: [PATCH 102/110] remove spec from laoder config --- internal/controller/discovery/core/types.go | 2 -- internal/controller/discovery/loaders.go | 9 +++++---- internal/controller/targetsource_controller.go | 16 ++++++++-------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 1dfcc9f..993c84e 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -3,7 +3,6 @@ package core import ( "context" - gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "k8s.io/apimachinery/pkg/types" ) @@ -21,7 +20,6 @@ type DiscoveryRegistryValue struct { type LoaderConfig struct { TargetsourceNN types.NamespacedName - Spec *gnmicv1alpha1.TargetSourceSpec ChunkSize int AcceptPush bool } diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index 487c76b..d179d3e 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -3,18 +3,19 @@ package discovery import ( "fmt" + gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1" "github.com/gnmic/operator/internal/controller/discovery/core" http "github.com/gnmic/operator/internal/controller/discovery/loaders/http" ) // NewLoader creates a loader by name -func NewLoader(cfg core.LoaderConfig) (core.Loader, error) { +func NewLoader(cfg core.LoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) { switch { - case cfg.Spec.Provider.HTTP != nil: - cfg.AcceptPush = cfg.Spec.Provider.HTTP.AcceptPush + case spec.Provider.HTTP != nil: + cfg.AcceptPush = spec.Provider.HTTP.AcceptPush return http.New(cfg), nil - case cfg.Spec.Provider.Consul != nil: + case spec.Provider.Consul != nil: return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) default: return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 2ba18a2..1cf962d 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -168,11 +168,16 @@ func (r *TargetSourceReconciler) startDiscovery( ) error { targetChannel := make(chan []discoveryTypes.DiscoveryMessage, r.BufferSize) ctx, cancel := context.WithCancel(context.Background()) + loaderConfig := discoveryTypes.LoaderConfig{ + TargetsourceNN: key, + ChunkSize: r.ChunkSize, + } // Register discovery runtime of targetsource if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{ - Channel: targetChannel, - Stop: cancel, + Channel: targetChannel, + Stop: cancel, + LoaderConfig: &loaderConfig, }); err != nil { return err } @@ -205,12 +210,7 @@ func (r *TargetSourceReconciler) startDiscovery( }() // Start target loader - loaderConfig := discoveryTypes.LoaderConfig{ - TargetsourceNN: key, - Spec: &targetSource.Spec, - ChunkSize: r.ChunkSize, - } - loader, err := discovery.NewLoader(loaderConfig) + loader, err := discovery.NewLoader(loaderConfig, &targetSource.Spec) if err != nil { logger.Error(err, "Target loader could not be created") cleanup() From 97849ae9d9a7afdc13aab966882433f0a59f0f7c Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 15:08:55 +0000 Subject: [PATCH 103/110] update LoaderConfig in registry --- internal/apiserver/apiserver.go | 2 +- internal/controller/discovery/core/types.go | 4 +- internal/controller/discovery/loaders.go | 8 ++-- .../discovery/loaders/http/loader.go | 20 +++++----- .../controller/targetsource_controller.go | 40 +++++++++++-------- 5 files changed, 40 insertions(+), 34 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 705b277..5eb88b8 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -5,8 +5,8 @@ import ( "net/http" "github.com/gnmic/operator/internal/controller" - "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery" + "github.com/gnmic/operator/internal/controller/discovery/core" "k8s.io/apimachinery/pkg/types" ) diff --git a/internal/controller/discovery/core/types.go b/internal/controller/discovery/core/types.go index 993c84e..99605b9 100644 --- a/internal/controller/discovery/core/types.go +++ b/internal/controller/discovery/core/types.go @@ -15,10 +15,10 @@ type DiscoveryRegistryValue struct { // Stop cancels the discovery context associated with this registry entry Stop context.CancelFunc - LoaderConfig *LoaderConfig + CommonLoaderConfig *CommonLoaderConfig } -type LoaderConfig struct { +type CommonLoaderConfig struct { TargetsourceNN types.NamespacedName ChunkSize int AcceptPush bool diff --git a/internal/controller/discovery/loaders.go b/internal/controller/discovery/loaders.go index d179d3e..7f2c656 100644 --- a/internal/controller/discovery/loaders.go +++ b/internal/controller/discovery/loaders.go @@ -9,16 +9,16 @@ import ( ) // NewLoader creates a loader by name -func NewLoader(cfg core.LoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) { +func NewLoader(cfg core.CommonLoaderConfig, spec *gnmicv1alpha1.TargetSourceSpec) (core.Loader, core.CommonLoaderConfig, error) { switch { case spec.Provider.HTTP != nil: cfg.AcceptPush = spec.Provider.HTTP.AcceptPush - return http.New(cfg), nil + return http.New(cfg), cfg, nil case spec.Provider.Consul != nil: - return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) + return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) default: - return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) + return nil, cfg, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", cfg.TargetsourceNN) } } diff --git a/internal/controller/discovery/loaders/http/loader.go b/internal/controller/discovery/loaders/http/loader.go index 17812aa..3325adb 100644 --- a/internal/controller/discovery/loaders/http/loader.go +++ b/internal/controller/discovery/loaders/http/loader.go @@ -13,12 +13,12 @@ import ( ) type Loader struct { - cfg core.LoaderConfig + commonCfg core.CommonLoaderConfig } // New instantiates the http loader with the provided config -func New(cfg core.LoaderConfig) core.Loader { - return &Loader{cfg: cfg} +func New(cfg core.CommonLoaderConfig) core.Loader { + return &Loader{commonCfg: cfg} } func (l *Loader) Name() string { @@ -29,13 +29,13 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er logger := log.FromContext(ctx).WithValues( "component", "loader", "name", l.Name(), - "targetsource", l.cfg.TargetsourceNN, + "targetsource", l.commonCfg.TargetsourceNN, ) logger.Info( "HTTP loader started", - "targetsource", l.cfg.TargetsourceNN.Name, - "namespace", l.cfg.TargetsourceNN.Namespace, + "targetsource", l.commonCfg.TargetsourceNN.Name, + "namespace", l.commonCfg.TargetsourceNN.Namespace, ) // Only for debugging: emit a static snapshot every 30 seconds @@ -50,21 +50,21 @@ func (l *Loader) Run(ctx context.Context, out chan<- []core.DiscoveryMessage) er case <-ticker.C: // Example snapshot (placeholder) - snapshotID := fmt.Sprintf("%s-%s-%s", l.cfg.TargetsourceNN.Namespace, l.cfg.TargetsourceNN.Name, uuid.NewString()) + snapshotID := fmt.Sprintf("%s-%s-%s", l.commonCfg.TargetsourceNN.Namespace, l.commonCfg.TargetsourceNN.Name, uuid.NewString()) targets := []core.DiscoveredTarget{ { Name: "ceos1", Address: "clab-3-nodes-ceos1:6030", - Labels: map[string]string{"TargetSource": l.cfg.TargetsourceNN.String()}, + Labels: map[string]string{"TargetSource": l.commonCfg.TargetsourceNN.String()}, }, { Name: "leaf1", Address: "clab-3-nodes-leaf1:57400", - Labels: map[string]string{"TargetSource": l.cfg.TargetsourceNN.String()}, + Labels: map[string]string{"TargetSource": l.commonCfg.TargetsourceNN.String()}, }, } - if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.cfg.ChunkSize); err != nil { + if err := loaderUtils.SendSnapshot(ctx, out, targets, snapshotID, l.commonCfg.ChunkSize); err != nil { return err } } diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 1cf962d..8207103 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -168,20 +168,11 @@ func (r *TargetSourceReconciler) startDiscovery( ) error { targetChannel := make(chan []discoveryTypes.DiscoveryMessage, r.BufferSize) ctx, cancel := context.WithCancel(context.Background()) - loaderConfig := discoveryTypes.LoaderConfig{ + loaderConfig := discoveryTypes.CommonLoaderConfig{ TargetsourceNN: key, ChunkSize: r.ChunkSize, } - // Register discovery runtime of targetsource - if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{ - Channel: targetChannel, - Stop: cancel, - LoaderConfig: &loaderConfig, - }); err != nil { - return err - } - // Cleanup function to cleanup discovery runtime of targetsource cleanup := func() { cancel() @@ -189,13 +180,34 @@ func (r *TargetSourceReconciler) startDiscovery( close(targetChannel) } - // Start message processor messageProcessor := discovery.NewMessageProcessor( r.Client, r.Scheme, targetSource, targetChannel, ) + loader, loaderConfig, err := discovery.NewLoader(discoveryTypes.CommonLoaderConfig{ + TargetsourceNN: key, + ChunkSize: r.ChunkSize, + }, + &targetSource.Spec, + ) + if err != nil { + logger.Error(err, "Target loader could not be created") + cleanup() + return err + } + + // Register discovery runtime of targetsource + if err := r.DiscoveryRegistry.Register(key, discoveryTypes.DiscoveryRegistryValue{ + Channel: targetChannel, + Stop: cancel, + CommonLoaderConfig: &loaderConfig, + }); err != nil { + return err + } + + // Start message processor go func() { logger.Info("Message processor started") @@ -210,12 +222,6 @@ func (r *TargetSourceReconciler) startDiscovery( }() // Start target loader - loader, err := discovery.NewLoader(loaderConfig, &targetSource.Spec) - if err != nil { - logger.Error(err, "Target loader could not be created") - cleanup() - return err - } go func() { if err := loader.Run(ctx, targetChannel); err != nil { logger.Error(err, "Target loader exited unexpectedly") From 953aaa75a93eccb567d5e31693c71c5309ae3a6a Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Fri, 1 May 2026 15:39:18 +0000 Subject: [PATCH 104/110] fix after merge --- internal/apiserver/targets.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/apiserver/targets.go b/internal/apiserver/targets.go index 2e434b9..7a55feb 100644 --- a/internal/apiserver/targets.go +++ b/internal/apiserver/targets.go @@ -22,7 +22,7 @@ func (a *APIServer) CreateTargets(c *gin.Context) { } registry, ok := a.DiscoveryRegistry.Get(key) - if !ok || (registry.LoaderConfig.AcceptPush != true) { + if !ok || (registry.CommonLoaderConfig.AcceptPush != true) { c.JSON(http.StatusNotFound, gin.H{ "error": "TargetSource not active or does not exist", }) From f683a44cb7e441c23e554f486034deb53b6c4a09 Mon Sep 17 00:00:00 2001 From: Janooski Date: Mon, 4 May 2026 09:06:52 +0000 Subject: [PATCH 105/110] comments from weekly --- internal/apiserver/apiserver.go | 85 ++---------------------------- internal/apiserver/helpers.go | 91 +++++++++++++++++++++++++++++++++ internal/apiserver/openapi.yaml | 2 +- 3 files changed, 97 insertions(+), 81 deletions(-) create mode 100644 internal/apiserver/helpers.go diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 8c01486..63ed1e1 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -4,6 +4,8 @@ package apiserver // To generate code, install openapi-codegen from https://github.com/oapi-codegen/oapi-codegen) // Then use: go generate ./internal/apiserver +// kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 + import ( "context" "net/http" @@ -40,13 +42,12 @@ func New(addr string, clusterReconciler *controller.ClusterReconciler, chunkSize clusterReconciler: clusterReconciler, chunkSize: chunkSize, } + // /api/v1/:namespace/target-source/:target_source_name apiBaseURL := "/api/v1/:namespace/:gNMIcControllerName" RegisterHandlersWithOptions(router, a, GinServerOptions{BaseURL: apiBaseURL}) return a, nil } -// kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 - // GetClusterPlan returns cluster plan func (a *APIServer) GetClusterPlan(c *gin.Context) { url := parseURI(c) @@ -74,86 +75,10 @@ func (a *APIServer) CreateTargets(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource " + payloadTargets.TargetSourceNameSpace + " / " + payloadTargets.TargetSourceName + " does not exist"}) return } - + // make sure channel is not closed if targetsource in registry is deleted + // timeout for sending to the channel targets := createDiscoveryEvent(payloadTargets) // fmt.Printf("core.DiscoveryEvent was created: %v", targets) core.SendEvents(context.Background(), ch, targets, a.chunkSize) c.JSON(http.StatusOK, payloadTargets) } - -// createDiscoveryEvent creates object of type core.DiscoveryEvent -func createDiscoveryEvent(payloadTargets Targets) []core.DiscoveryEvent { - targets := []core.DiscoveryEvent{} - if len(payloadTargets.TargetList) > 0 { - for i, target := range payloadTargets.TargetList { - if target.Address == "" || target.Name == "" || target.Operation == "" { - // no REST API return here as not all targets might - logger.Warn("Target receieved at index", i, " by pull interface does not contain Address, Name or Operation and is skipped.") - break - } - if target.Operation.Valid() != true { - logger.Warn("Target receieved at index", i, " by pull interface has invalid Operation.") - break - } - - targets = append(targets, core.DiscoveryEvent{ - Target: core.DiscoveredTarget{ - Name: target.Name, - Address: target.Address, - Labels: convertTargetLabelsToMap(target), - }, - Event: getEvent(target), - }) - } - } - return targets -} - -// getKey returns key for used to identify correct channel in DiscoveryRegistry -func getKey(payloadTargets Targets) types.NamespacedName { - key := types.NamespacedName{ - Namespace: payloadTargets.TargetSourceNameSpace, - Name: payloadTargets.TargetSourceName, - } - return key -} - -// convertTargetLabelsToMap converts target.Labels to map. -func convertTargetLabelsToMap(target Target) map[string]string { - labelToMap := make(map[string]string) - if target.Labels != nil { - for _, tag := range *target.Labels { - if tag.Key == nil || tag.Value == nil || *tag.Key == "" { - continue - } - labelToMap[*tag.Key] = *tag.Value - } - } - return labelToMap -} - -// getEvent converts target.Operation to core.Operation. -func getEvent(target Target) core.EventAction { - event := core.CREATE - switch target.Operation { - case Created: - event = core.UPDATE - case Updated: - event = core.UPDATE - case Deleted: - event = core.DELETE - default: - logger.Warn("Received invalid Operation flag") - } - return event -} - -// parseURI parses URI to urlStruct. -func parseURI(c *gin.Context) (url urlStruct) { - var u urlStruct - if err := c.ShouldBindUri(&u); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - return u -} diff --git a/internal/apiserver/helpers.go b/internal/apiserver/helpers.go new file mode 100644 index 0000000..382b44a --- /dev/null +++ b/internal/apiserver/helpers.go @@ -0,0 +1,91 @@ +package apiserver + +import ( + "fmt" + "net/http" + + "github.com/bytedance/gopkg/util/logger" + "github.com/gin-gonic/gin" + "github.com/gnmic/operator/internal/controller/discovery/core" + "k8s.io/apimachinery/pkg/types" +) + +// createDiscoveryEvent creates object of type core.DiscoveryEvent +func createDiscoveryEvent(payloadTargets Targets) []core.DiscoveryEvent { + targets := []core.DiscoveryEvent{} + if len(payloadTargets.TargetList) > 0 { + for i, target := range payloadTargets.TargetList { + if target.Name == "" { + // no REST API return here as not all targets might be incomplete + err := fmt.Errorf("Target receieved at index %d by pull interface has no Name and is skipped.", i) + logger.Error(err, "Failed creating DiscoveryEvent") + break + } + if target.Address == "" { + err := fmt.Errorf("Target receieved at index %d by pull interface has no Address and is skipped.", i) + logger.Error(err, "Failed creating DiscoveryEvent") + break + } + + targets = append(targets, core.DiscoveryEvent{ + Target: core.DiscoveredTarget{ + Name: target.Name, + Address: target.Address, + Labels: convertTargetLabelsToMap(target), + }, + Event: getEvent(target, i), + }) + } + } + return targets +} + +// getKey returns key for used to identify correct channel in DiscoveryRegistry +func getKey(payloadTargets Targets) types.NamespacedName { + key := types.NamespacedName{ + Namespace: payloadTargets.TargetSourceNameSpace, + Name: payloadTargets.TargetSourceName, + } + return key +} + +// convertTargetLabelsToMap converts target.Labels to map. +func convertTargetLabelsToMap(target Target) map[string]string { + labelToMap := make(map[string]string) + if target.Labels != nil { + for _, tag := range *target.Labels { + if tag.Key == nil || tag.Value == nil || *tag.Key == "" { + continue + } + labelToMap[*tag.Key] = *tag.Value + } + } + return labelToMap +} + +// getEvent converts target.Operation to core.Operation. +func getEvent(target Target, index int) core.EventAction { + event := core.CREATE + switch target.Operation { + case Created: + event = core.UPDATE + case Updated: + event = core.UPDATE + case Deleted: + event = core.DELETE + default: + err := fmt.Errorf("Target receieved at index %d by pull interface has no valid Operation and is skipped.", index) + logger.Error(err, "Failed creating DiscoveryEvent") + } + return event +} + +// parseURI parses URI to urlStruct. +func parseURI(c *gin.Context) (url urlStruct) { + var u urlStruct + if err := c.ShouldBindUri(&u); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + return u +} diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index 093c1a4..99a9a87 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -66,7 +66,7 @@ components: - created - updated - deleted - Tag: + Tag: # tag will be removed, moved directly under labels type: object properties: key: From fe31b0d3c43225f8dc8e80f0cc2bc6e23b2c4138 Mon Sep 17 00:00:00 2001 From: Janooski Date: Mon, 4 May 2026 09:08:36 +0000 Subject: [PATCH 106/110] update manifests --- api/v1alpha1/zz_generated.deepcopy.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 608d47e..3656535 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -826,6 +826,11 @@ func (in *ProviderSpec) DeepCopyInto(out *ProviderSpec) { *out = new(HTTPConfig) **out = **in } + if in.PULL != nil { + in, out := &in.PULL, &out.PULL + *out = new(WebhookSpec) + (*in).DeepCopyInto(*out) + } if in.Consul != nil { in, out := &in.Consul, &out.Consul *out = new(ConsulConfig) From 08273d0ad1f37bdb4878b7f5a8e5f073aaf6237f Mon Sep 17 00:00:00 2001 From: Janooski Date: Mon, 4 May 2026 11:49:34 +0000 Subject: [PATCH 107/110] udpate contract --- internal/apiserver/apiserver.go | 18 ++++++-------- internal/apiserver/gen.go | 31 ++++++++++++------------ internal/apiserver/helpers.go | 12 ++++----- internal/apiserver/openapi.yaml | 43 ++++++++++++++------------------- internal/apiserver/temp.md | 19 ++++++++++++++- 5 files changed, 64 insertions(+), 59 deletions(-) diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 81b7e8f..8abd2ef 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -31,8 +31,8 @@ type APIServer struct { } type urlStruct struct { - namespace string `uri:"namespace" binding:"required"` - gNMIcControllerName string `uri:"gNMIcControllerName" binding:"required"` + Namespace string `uri:"namespace" binding:"required"` + Name string `uri:"name" binding:"required"` } func New( @@ -55,14 +55,10 @@ func New( DiscoveryRegistry: discoveryRegistry, chunzSize: discoveryChunksize, } - // apiBaseURL := "/api/v1/:namespace/:gNMIcClusterName" - // RegisterHandlersWithOptions(router, a, GinServerOptions{BaseURL: apiBaseURL}) a.routes() return a, nil } -// kubectl port-forward -n gnmic-system svc/gnmic-controller-manager-api 8082:8082 --address=0.0.0.0 - func (a *APIServer) Router() *gin.Engine { return a.router } @@ -75,7 +71,7 @@ func (a *APIServer) routes() { // GetClusterPlan returns cluster plan func (a *APIServer) GetClusterPlan(c *gin.Context) { url := parseURI(c) - plan, err := a.clusterReconciler.GetClusterPlan(url.namespace, url.gNMIcControllerName) + plan, err := a.clusterReconciler.GetClusterPlan(url.Namespace, url.Name) if err != nil { c.String(404, err.Error()) return @@ -86,17 +82,17 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { // CreateTargets binds payload to payloadTargets struct defined in openapi contract. Creates a []core.DiscoveryEvent sends it to the core package. func (a *APIServer) CreateTargets(c *gin.Context) { logger.Info("received POST request for CreateTargets.") - + url := parseURI(c) var payloadTargets Targets if err := c.ShouldBind(&payloadTargets); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - registry, ok := a.DiscoveryRegistry.Get(getKey(payloadTargets)) + registry, ok := a.DiscoveryRegistry.Get(getKey(url)) if !ok { - logger.Error("TargetSource ", payloadTargets.TargetSourceNameSpace, "/", payloadTargets.TargetSourceName, "does not exist.") - c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource " + payloadTargets.TargetSourceNameSpace + " / " + payloadTargets.TargetSourceName + " does not exist"}) + logger.Error("TargetSource ", url.Namespace, "/", url.Name, "does not exist.") + c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource " + url.Namespace + " / " + url.Name + " does not exist"}) return } // make sure channel is not closed if targetsource in registry is deleted diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index 07a82c2..5811b9b 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -37,8 +37,8 @@ func (e TargetOperation) Valid() bool { } } -// Tag defines model for Tag. -type Tag struct { +// Label defines model for Label. +type Label struct { Key *string `json:"key,omitempty"` Value *string `json:"value,omitempty"` } @@ -46,7 +46,7 @@ type Tag struct { // Target defines model for Target. type Target struct { Address string `json:"address"` - Labels *[]Tag `json:"labels,omitempty"` + Labels *[]Label `json:"labels,omitempty"` Name string `json:"name"` Operation TargetOperation `json:"operation"` Profile *string `json:"profile,omitempty"` @@ -56,11 +56,10 @@ type Target struct { type TargetOperation string // Targets defines model for Targets. -type Targets struct { - TargetList []Target `json:"TargetList"` - TargetSourceName string `json:"TargetSourceName"` - TargetSourceNameSpace string `json:"targetSourceNameSpace"` -} +type Targets = []Target + +// CreateTargetsJSONRequestBody defines body for CreateTargets for application/json ContentType. +type CreateTargetsJSONRequestBody = Targets // ServerInterface represents all server handlers. type ServerInterface interface { @@ -141,14 +140,14 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/5STz2vbMBTH/xXxtqOJ3e7m2yijFLauLLmNHlT5JVEnS9rTU8EU/+9DkpMmtQPdSc96", - "vz9f+RWU672zaDlA+wpB7bGX2dzIXTo8OY/EGvPlHxzSwYNHaCEwabuDsYIXaSIueMbqcOOenlFxit1I", - "2iHPa8uuIwxhsb6RT2iySzP22fhMuIUWPtVvG9TT+HWa/a21JJJD+rayx8XyaQzJ2tnkRRt7aH+DIpSM", - "HVQQfTdZHRpM1mM1L+LJbbW5QIHwb9SEXSqcx6iO+562f7zIK8yBFcd3Hfg/wGT2C2yKZ+0iKby/xInf", - "Ba29VB9YeDltoWd1utOcRaqr7dbljpoTa9jd/7hT4mcm6Ej8+rbeiK8Pd1DBC1LIkkKzalZXk85Weg0t", - "fFldrxqowEveZ2p1kfsUtitgj+rcddDCzVlY2jN4Z0NR5Lq5SodyltHmZOm90Sqn18+hPLAixiU9P6hh", - "mFE+3C9z6zAo0r488kOsmN64CFEpDGEbjRly4RD7XtJwXFjwlKGt4D2Kc+w5pfZG5v2mv/uc2y3yjYmB", - "kR5S2Axck47zIU/iBSFHsti9G+4WWagSJnL7cRzHfwEAAP//JDP+7tUEAAA=", + "H4sIAAAAAAAC/7RTO2/cMAz+KwLb0Tg76eatDYogQB9Bm63IoEi8O6WypFJUACPwfy9EO/foXYAsnURL", + "JL8H6WcwcUgxYOAM/TNks8VBS/hFP6CvQaKYkNihXP/GsR48JoQeMpMLG5gaeNK+4JmXqXm5iQ+PaLjm", + "3mnaIJ/21tYS5ny2v6905MkxDhK8J1xDD+/avYZ2EdDO7PfgmkiP9TvoAc8CVCKaXQz1FUMZoP8FhlAz", + "WmigJLtEFj3W6L45bZIorp1/xQfCP8UR2tpYaDQ7xYfw96869nb5i8Mn+isNF9ZRCDquTGHz7euNUd8F", + "P5L68fnnnfp4ewMNPCFlMQS6Vbe6WFwKOjno4cPqctVBA0nzVvi0s1kHVFPMMuWdthsLPVwdpc22YOZP", + "0cpmmRgYg9TplLwzUtk+5nkys8K36c+z4L3vTAXlIqcY8rx0l93F/4G1mA25NK/UywTVslEqF2Mw53Xx", + "fpTtyGUYNI07gxQvFS4o3qI6HpOUtMlrYbf8Tcc+XyNf+ZIZ6bamncju6nFM8iBfEXKhgPYfctfIysxp", + "SuCnaZr+BgAA//+GH+FbRwQAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/apiserver/helpers.go b/internal/apiserver/helpers.go index 5114dfe..37eb12e 100644 --- a/internal/apiserver/helpers.go +++ b/internal/apiserver/helpers.go @@ -11,10 +11,10 @@ import ( ) // createDiscoveryEvent creates object of type core.DiscoveryEvent -func createDiscoveryEvent(payloadTargets Targets) []core.DiscoveryEvent { +func createDiscoveryEvent(payloadTargets []Target) []core.DiscoveryEvent { targets := []core.DiscoveryEvent{} - if len(payloadTargets.TargetList) > 0 { - for i, target := range payloadTargets.TargetList { + if len(payloadTargets) > 0 { + for i, target := range payloadTargets { if target.Name == "" { // no REST API return here as not all targets might be incomplete err := fmt.Errorf("Target receieved at index %d by pull interface has no Name and is skipped.", i) @@ -41,10 +41,10 @@ func createDiscoveryEvent(payloadTargets Targets) []core.DiscoveryEvent { } // getKey returns key for used to identify correct channel in DiscoveryRegistry -func getKey(payloadTargets Targets) types.NamespacedName { +func getKey(u urlStruct) types.NamespacedName { key := types.NamespacedName{ - Namespace: payloadTargets.TargetSourceNameSpace, - Name: payloadTargets.TargetSourceName, + Namespace: u.Namespace, + Name: u.Name, } return key } diff --git a/internal/apiserver/openapi.yaml b/internal/apiserver/openapi.yaml index 99a9a87..e7745c9 100644 --- a/internal/apiserver/openapi.yaml +++ b/internal/apiserver/openapi.yaml @@ -14,38 +14,37 @@ paths: post: summary: "Create targets in the gNMIc Operator" operationId: "createTargets" + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/Targets' responses: '201': description: "Targets created successfully" content: application/json: schema: - type: object - required: - - Targets - properties: - Targets: - $ref: '#/components/schemas/Targets' + $ref: '#/components/schemas/Targets' components: schemas: Targets: + type: array + items: + $ref: '#/components/schemas/Target' + + Label: type: object - required: - - targetSourceNameSpace - - TargetSourceName - - TargetList properties: - targetSourceNameSpace: + key: type: string - TargetSourceName: + value: type: string - TargetList: - type: array - items: - $ref: '#/components/schemas/Target' + Target: type: object - required: + required: - name - address - operation @@ -59,17 +58,11 @@ components: labels: type: array items: - $ref: '#/components/schemas/Tag' + $ref: '#/components/schemas/Label' operation: type: string enum: - created - updated - deleted - Tag: # tag will be removed, moved directly under labels - type: object - properties: - key: - type: string - value: - type: string \ No newline at end of file + \ No newline at end of file diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index c70f463..b1cf607 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -1,5 +1,22 @@ +curl -X POST "http://localhost:8082/api/v1/default/target-source/http-discovery/createTargets" \ + -H "Content-Type: application/json" \ + -d '[ + { + "address": "1.1.1.1", + "name": "Router1", + "operation": "created", + "profile": "defaultProfile", + "labels": [ + { "key": "tags", "value": "tag1, tag2" } + ] + } + ]' + + +# old (before 4th of may) + ## CURL request -curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"webhook-test", "TargetSourceNameSpace":"default", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"created","Profile":"defaultProfile", "Labels": [{"key": "tags", "value": "tag1, tag2"}]}]}' +curl -X POST "http://localhost:8082/api/v1/gnmic-system/target-source/netbox/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"http-discovery", "TargetSourceNameSpace":"default", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"created","Profile":"defaultProfile", "Labels": [{"key": "tags", "value": "tag1, tag2"}]}]}' ## Empty TargetList curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' From 426e27ae1e39a33a963d6e24ea25362b56683f6f Mon Sep 17 00:00:00 2001 From: Daniel Schatzmann Date: Mon, 4 May 2026 18:15:49 +0000 Subject: [PATCH 108/110] fix: use defined variable --- internal/controller/targetsource_controller.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/controller/targetsource_controller.go b/internal/controller/targetsource_controller.go index 8207103..c65e254 100644 --- a/internal/controller/targetsource_controller.go +++ b/internal/controller/targetsource_controller.go @@ -186,10 +186,7 @@ func (r *TargetSourceReconciler) startDiscovery( targetSource, targetChannel, ) - loader, loaderConfig, err := discovery.NewLoader(discoveryTypes.CommonLoaderConfig{ - TargetsourceNN: key, - ChunkSize: r.ChunkSize, - }, + loader, loaderConfig, err := discovery.NewLoader(loaderConfig, &targetSource.Spec, ) if err != nil { From 84012af63692c50f1b94e1d8c6f85389db7216a9 Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 6 May 2026 09:44:55 +0000 Subject: [PATCH 109/110] change logger --- internal/apiserver/apiserver.go | 38 +++++++++++++++++++---- internal/apiserver/apiserver_test.go | 2 ++ internal/apiserver/helpers.go | 31 +++++++++---------- internal/apiserver/temp.md | 45 ++++++++-------------------- 4 files changed, 63 insertions(+), 53 deletions(-) create mode 100644 internal/apiserver/apiserver_test.go diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 8abd2ef..c6e7d4b 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -8,15 +8,17 @@ package apiserver import ( "context" + "fmt" "net/http" - "github.com/bytedance/gopkg/util/logger" "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller" "github.com/gnmic/operator/internal/controller/discovery" "github.com/gnmic/operator/internal/controller/discovery/core" "github.com/gnmic/operator/internal/controller/discovery/loaders/utils" + "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/log" ) type APIServer struct { @@ -28,6 +30,7 @@ type APIServer struct { core.DiscoveryRegistryValue, ] chunzSize int + logger logr.Logger } type urlStruct struct { @@ -45,6 +48,7 @@ func New( discoveryChunksize int, ) (*APIServer, error) { router := gin.Default() + logger := log.Log.WithValues("component", "api-server") a := &APIServer{ Server: &http.Server{ Addr: addr, @@ -54,12 +58,15 @@ func New( clusterReconciler: clusterReconciler, DiscoveryRegistry: discoveryRegistry, chunzSize: discoveryChunksize, + logger: logger, } + logger.Info("API server initialized", "addr", addr, "chunkSize", discoveryChunksize) a.routes() return a, nil } func (a *APIServer) Router() *gin.Engine { + gin.SetMode(gin.ReleaseMode) // gin logs return a.router } @@ -71,34 +78,55 @@ func (a *APIServer) routes() { // GetClusterPlan returns cluster plan func (a *APIServer) GetClusterPlan(c *gin.Context) { url := parseURI(c) + logger := log.FromContext(c.Request.Context()).WithValues( + "component", "apiserver", + "namespace", url.Namespace, + "cluster", url.Name, + ) + logger.Info("Received GET request for GetClusterPlan") + plan, err := a.clusterReconciler.GetClusterPlan(url.Namespace, url.Name) if err != nil { + logger.Error(err, "Failed to get cluster plan") c.String(404, err.Error()) return } + logger.Info("Successfully returned cluster plan") c.JSON(200, plan) } // CreateTargets binds payload to payloadTargets struct defined in openapi contract. Creates a []core.DiscoveryEvent sends it to the core package. func (a *APIServer) CreateTargets(c *gin.Context) { - logger.Info("received POST request for CreateTargets.") url := parseURI(c) + logger := log.FromContext(c.Request.Context()).WithValues( + "component", "apiserver", + "namespace", url.Namespace, + "targetsource", url.Name, + ) + logger.Info("Received POST request for CreateTargets") + var payloadTargets Targets if err := c.ShouldBind(&payloadTargets); err != nil { + logger.Error(err, "Failed to bind request payload") c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } registry, ok := a.DiscoveryRegistry.Get(getKey(url)) if !ok { - logger.Error("TargetSource ", url.Namespace, "/", url.Name, "does not exist.") + err := fmt.Errorf("targetSource %s/%s does not exist", url.Namespace, url.Name) + logger.Error(err, "TargetSource lookup failed") c.JSON(http.StatusBadRequest, gin.H{"error": "TargetSource " + url.Namespace + " / " + url.Name + " does not exist"}) return } // make sure channel is not closed if targetsource in registry is deleted // timeout for sending to the channel - targets := createDiscoveryEvent(payloadTargets) - // fmt.Printf("core.DiscoveryEvent was created: %v", targets) + targets, err := createDiscoveryEvent(payloadTargets) + if err != nil{ + logger.Error(err, "failed creating discoveryEvent") + c.JSON(http.StatusBadRequest, gin.H{"error": err}) + } utils.SendEvents(context.Background(), registry.Channel, targets, a.chunzSize) + logger.Info("CreateTargets request processed successfully", "count", len(targets)) c.JSON(http.StatusOK, payloadTargets) } diff --git a/internal/apiserver/apiserver_test.go b/internal/apiserver/apiserver_test.go new file mode 100644 index 0000000..c129d1a --- /dev/null +++ b/internal/apiserver/apiserver_test.go @@ -0,0 +1,2 @@ +package apiserver + diff --git a/internal/apiserver/helpers.go b/internal/apiserver/helpers.go index 37eb12e..cc556a4 100644 --- a/internal/apiserver/helpers.go +++ b/internal/apiserver/helpers.go @@ -4,27 +4,27 @@ import ( "fmt" "net/http" - "github.com/bytedance/gopkg/util/logger" "github.com/gin-gonic/gin" "github.com/gnmic/operator/internal/controller/discovery/core" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/log" ) // createDiscoveryEvent creates object of type core.DiscoveryEvent -func createDiscoveryEvent(payloadTargets []Target) []core.DiscoveryEvent { +func createDiscoveryEvent(payloadTargets []Target) ([]core.DiscoveryEvent, error) { targets := []core.DiscoveryEvent{} + if len(payloadTargets) > 0 { for i, target := range payloadTargets { if target.Name == "" { - // no REST API return here as not all targets might be incomplete - err := fmt.Errorf("Target receieved at index %d by pull interface has no Name and is skipped.", i) - logger.Error(err, "Failed creating DiscoveryEvent") - break + return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Name and is skipped.", i) } if target.Address == "" { - err := fmt.Errorf("Target receieved at index %d by pull interface has no Address and is skipped.", i) - logger.Error(err, "Failed creating DiscoveryEvent") - break + return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Address and is skipped.", i) + } + event, err := getEvent(target, i) + if err != nil { + return nil, err } targets = append(targets, core.DiscoveryEvent{ @@ -33,11 +33,11 @@ func createDiscoveryEvent(payloadTargets []Target) []core.DiscoveryEvent { Address: target.Address, Labels: convertTargetLabelsToMap(target), }, - Event: getEvent(target, i), + Event: event, }) } } - return targets + return targets, nil } // getKey returns key for used to identify correct channel in DiscoveryRegistry @@ -64,7 +64,7 @@ func convertTargetLabelsToMap(target Target) map[string]string { } // getEvent converts target.Operation to core.Operation. -func getEvent(target Target, index int) core.EventAction { +func getEvent(target Target, index int) (core.EventAction, error) { event := core.EventApply switch target.Operation { case Created: @@ -74,16 +74,17 @@ func getEvent(target Target, index int) core.EventAction { case Deleted: event = core.EventDelete default: - err := fmt.Errorf("Target receieved at index %d by pull interface has no valid Operation and is skipped.", index) - logger.Error(err, "Failed creating DiscoveryEvent") + return event, fmt.Errorf("Target receieved at index %d by pull interface has no valid Operation and is skipped.", index) } - return event + return event, nil } // parseURI parses URI to urlStruct. func parseURI(c *gin.Context) (url urlStruct) { + logger := log.FromContext(c.Request.Context()).WithValues("component", "apiserver", "action", "parse-uri") var u urlStruct if err := c.ShouldBindUri(&u); err != nil { + logger.Error(err, "Failed to bind request URI") c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } diff --git a/internal/apiserver/temp.md b/internal/apiserver/temp.md index b1cf607..afc0eb9 100644 --- a/internal/apiserver/temp.md +++ b/internal/apiserver/temp.md @@ -13,39 +13,18 @@ curl -X POST "http://localhost:8082/api/v1/default/target-source/http-discovery/ ]' -# old (before 4th of may) - -## CURL request -curl -X POST "http://localhost:8082/api/v1/gnmic-system/target-source/netbox/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"http-discovery", "TargetSourceNameSpace":"default", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"created","Profile":"defaultProfile", "Labels": [{"key": "tags", "value": "tag1, tag2"}]}]}' - -## Empty TargetList -curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace"}' - -## Empty Target in Target List -curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":""}]}' - -## Empty TargetSourceName -curl -X POST "http://localhost:8082/api/v1/gnmic-system/gnmic-controller-manager/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"create"}]}' - -## Wrong operation -curl -X POST "http://localhost:8082/api/v1/gnmic-system/cluster1/createTargets" -H "Content-Type: application/json" -d '{"TargetSourceName":"sourcename", "TargetSourceNameSpace":"namespace", "TargetList": [{"Address":"1.1.1.1", "Name": "Router1", "Operation":"notupdate","Profile":"defaultProfile", "tags": ["tag1", "tag2"]}]}' - - - -{ - "TargetSourceName": "webhook-test", - "TargetSourceNameSpace": "default", - "TargetList": [ +http://gnmic-controller-manager-api.gnmic-system.svc.cluster.local:8082/api/v1/default/target-source/http-discovery/createTargets +[ + { + "address": "{{ data.primary_ip4.address.split('/')[0] if data.primary_ip4 and data.primary_ip4.address else '' }}:{{ data.custom_fields.port }}", + "name": "{{ data.name }}", + "operation": "{{ event }}", + "profile": "{{ data.custom_fields.profile | default('') }}", + "labels": [ { - "name": "{{ data.name }}", - "address": "{{ data.primary_ip4.address.split('/')[0] if data.primary_ip4 else '' }}:{{ data.custom_fields.port }}", - "profile": "{{ data.custom_fields.profile | default('') }}", - "labels": [{ - "key": "tags", - "value": "{{ data.tags | map(attribute='name') | join(', ') }}" - }, - "operation": "{{ event }}" - ] + "Key": "tags", + "Value": "{{ data.tags | map(attribute='name') | join(', ') }}" } ] -} \ No newline at end of file + } +] \ No newline at end of file From 7fa3e5baf40d2f75115f15af20251e0671fc3af9 Mon Sep 17 00:00:00 2001 From: Janooski Date: Wed, 6 May 2026 12:46:39 +0000 Subject: [PATCH 110/110] unit test for helpers --- internal/apiserver/apiserver.go | 6 +- internal/apiserver/apiserver_test.go | 2 - internal/apiserver/gen.go | 6 +- internal/apiserver/helpers.go | 12 +- internal/apiserver/helpers_test.go | 265 +++++++++++++++++++++++++++ 5 files changed, 276 insertions(+), 15 deletions(-) delete mode 100644 internal/apiserver/apiserver_test.go create mode 100644 internal/apiserver/helpers_test.go diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index c6e7d4b..a46837f 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -47,6 +47,7 @@ func New( ], discoveryChunksize int, ) (*APIServer, error) { + gin.SetMode(gin.ReleaseMode) // To double-check router := gin.Default() logger := log.Log.WithValues("component", "api-server") a := &APIServer{ @@ -66,7 +67,6 @@ func New( } func (a *APIServer) Router() *gin.Engine { - gin.SetMode(gin.ReleaseMode) // gin logs return a.router } @@ -91,7 +91,6 @@ func (a *APIServer) GetClusterPlan(c *gin.Context) { c.String(404, err.Error()) return } - logger.Info("Successfully returned cluster plan") c.JSON(200, plan) } @@ -122,11 +121,10 @@ func (a *APIServer) CreateTargets(c *gin.Context) { // make sure channel is not closed if targetsource in registry is deleted // timeout for sending to the channel targets, err := createDiscoveryEvent(payloadTargets) - if err != nil{ + if err != nil { logger.Error(err, "failed creating discoveryEvent") c.JSON(http.StatusBadRequest, gin.H{"error": err}) } utils.SendEvents(context.Background(), registry.Channel, targets, a.chunzSize) - logger.Info("CreateTargets request processed successfully", "count", len(targets)) c.JSON(http.StatusOK, payloadTargets) } diff --git a/internal/apiserver/apiserver_test.go b/internal/apiserver/apiserver_test.go deleted file mode 100644 index c129d1a..0000000 --- a/internal/apiserver/apiserver_test.go +++ /dev/null @@ -1,2 +0,0 @@ -package apiserver - diff --git a/internal/apiserver/gen.go b/internal/apiserver/gen.go index 5811b9b..80f2c5f 100644 --- a/internal/apiserver/gen.go +++ b/internal/apiserver/gen.go @@ -43,8 +43,8 @@ type Label struct { Value *string `json:"value,omitempty"` } -// Target defines model for Target. -type Target struct { +// target defines model for target. +type target struct { Address string `json:"address"` Labels *[]Label `json:"labels,omitempty"` Name string `json:"name"` @@ -56,7 +56,7 @@ type Target struct { type TargetOperation string // Targets defines model for Targets. -type Targets = []Target +type Targets = []target // CreateTargetsJSONRequestBody defines body for CreateTargets for application/json ContentType. type CreateTargetsJSONRequestBody = Targets diff --git a/internal/apiserver/helpers.go b/internal/apiserver/helpers.go index cc556a4..487e371 100644 --- a/internal/apiserver/helpers.go +++ b/internal/apiserver/helpers.go @@ -11,16 +11,16 @@ import ( ) // createDiscoveryEvent creates object of type core.DiscoveryEvent -func createDiscoveryEvent(payloadTargets []Target) ([]core.DiscoveryEvent, error) { +func createDiscoveryEvent(payloadTargets []target) ([]core.DiscoveryEvent, error) { targets := []core.DiscoveryEvent{} if len(payloadTargets) > 0 { for i, target := range payloadTargets { if target.Name == "" { - return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Name and is skipped.", i) + return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Name.", i) } if target.Address == "" { - return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Address and is skipped.", i) + return nil, fmt.Errorf("Target receieved at index %d by pull interface has no Address.", i) } event, err := getEvent(target, i) if err != nil { @@ -50,7 +50,7 @@ func getKey(u urlStruct) types.NamespacedName { } // convertTargetLabelsToMap converts target.Labels to map. -func convertTargetLabelsToMap(target Target) map[string]string { +func convertTargetLabelsToMap(target target) map[string]string { labelToMap := make(map[string]string) if target.Labels != nil { for _, tag := range *target.Labels { @@ -64,7 +64,7 @@ func convertTargetLabelsToMap(target Target) map[string]string { } // getEvent converts target.Operation to core.Operation. -func getEvent(target Target, index int) (core.EventAction, error) { +func getEvent(target target, index int) (core.EventAction, error) { event := core.EventApply switch target.Operation { case Created: @@ -74,7 +74,7 @@ func getEvent(target Target, index int) (core.EventAction, error) { case Deleted: event = core.EventDelete default: - return event, fmt.Errorf("Target receieved at index %d by pull interface has no valid Operation and is skipped.", index) + return event, fmt.Errorf("Target receieved at index %d by pull interface has no valid Operation", index) } return event, nil } diff --git a/internal/apiserver/helpers_test.go b/internal/apiserver/helpers_test.go new file mode 100644 index 0000000..9bfa41b --- /dev/null +++ b/internal/apiserver/helpers_test.go @@ -0,0 +1,265 @@ +package apiserver + +import ( + "reflect" + "testing" + + "net/http" + "net/http/httptest" + + "github.com/gin-gonic/gin" + "github.com/gnmic/operator/internal/controller/discovery/core" + "k8s.io/apimachinery/pkg/types" +) + +func TestGetEventApply(t *testing.T) { + target := target{ + Address: "1.1.1.1", + Name: "routername", + Labels: &[]Label{}, + Operation: "created", + } + event, err := getEvent(target, 0) + if event != core.EventApply { + t.Errorf("getEvent(target) = %d, want core.EventApply", event) + } + if err != nil { + t.Errorf("getEvent(target) returns err: %s", err) + } +} + +func TestGetEventDelete(t *testing.T) { + target := target{ + Address: "1.1.1.1", + Name: "routername", + Labels: &[]Label{}, + Operation: "deleted", + } + event, err := getEvent(target, 0) + if event != core.EventDelete { + t.Errorf("getEvent(target) = %d, want core.EventDelete", event) + } + if err != nil { + t.Errorf("getEvent(target) returns err: %s", err) + } +} + +func TestGetEventEmptyOperation(t *testing.T) { + target := target{ + Address: "1.1.1.1", + Name: "routername", + Labels: &[]Label{}, + Operation: "", + } + event, err := getEvent(target, 0) + if err == nil { + t.Errorf("getEvent(target, 0) = %d, want error", event) + } +} + +func TestGetEventUpdate(t *testing.T) { + target := target{ + Address: "1.1.1.1", + Name: "routername", + Labels: &[]Label{}, + Operation: "updated", + } + event, err := getEvent(target, 0) + if event != core.EventApply { + t.Errorf("getEvent(target) = %d, want core.EventApply", event) + } + if err != nil { + t.Errorf("getEvent(target) returns err: %s", err) + } +} + +func TestGetKey(t *testing.T) { + u := urlStruct{ + Namespace: "default", + Name: "http-discovery", + } + expected := types.NamespacedName{ + Namespace: "default", + Name: "http-discovery", + } + result := getKey(u) + if result != expected { + t.Errorf("getKey(%v) = %v; want %v", u, result, expected) + } +} + +func TestConvertTargetLabelsToMapEmpty(t *testing.T) { + target := target{} + result := convertTargetLabelsToMap(target) + if len(result) != 0 { + t.Errorf("convertTargetLabelsToMap(target) = %v; want empty map", result) + } +} + +func TestConvertTargetLabelsToMap(t *testing.T) { + key := "Tag" + value := "TT1, TT2" + label := Label{ + Key: &key, + Value: &value, + } + target := target{ + Labels: &[]Label{label}, + } + expected := map[string]string{ + "Tag": "TT1, TT2", + } + result := convertTargetLabelsToMap(target) + if !reflect.DeepEqual(result, expected) { + t.Errorf("convertTargetLabelsToMap(target) = %v; want %v", result, expected) + } +} + +func TestConvertTargetLabelsToMapEmptyKey(t *testing.T) { + key := "" + value := "TT1, TT2" + label := Label{ + Key: &key, + Value: &value, + } + target := target{ + Labels: &[]Label{label}, + } + result := convertTargetLabelsToMap(target) + if len(result) != 0 { + t.Errorf("convertTargetLabelsToMap(target) = %v; want empty map", result) + } +} + +func TestConvertTargetLabelsToMapTwoEntries(t *testing.T) { + key := "Tag" + key2 := "Tag1" + value := "TT1, TT2" + value2 := "TT1" + label := Label{ + Key: &key, + Value: &value, + } + label2 := Label{ + Key: &key2, + Value: &value2, + } + target := target{ + Labels: &[]Label{label, label2}, + } + expected := map[string]string{ + "Tag": "TT1, TT2", + "Tag1": "TT1", + } + result := convertTargetLabelsToMap(target) + if !reflect.DeepEqual(result, expected) { + t.Errorf("convertTargetLabelsToMap(target) = %v; want %v", result, expected) + } +} + +func TestCreateDiscoveryEvent(t *testing.T) { + targets := []target{{ + Address: "1.1.1.1", + Name: "routername", + Labels: &[]Label{}, + Operation: "updated"}} + + expected := []core.DiscoveryEvent{ + { + Target: core.DiscoveredTarget{ + Name: "routername", + Address: "1.1.1.1", + Labels: map[string]string{}, + }, + Event: core.EventApply, + }, + } + result, _ := createDiscoveryEvent(targets) + if !reflect.DeepEqual(result, expected) { + t.Errorf("createDiscoveryEvent(targets) = %v; want %v", result, expected) + } +} + +func TestCreateDiscoveryEventEmptyName(t *testing.T) { + targets := []target{{ + Address: "1.1.1.1", + Name: "", + Labels: &[]Label{}, + Operation: "updated"}} + + result, err := createDiscoveryEvent(targets) + if err == nil { + t.Errorf("createDiscoveryEvent(targets) returns %v, want missing name error", result) + } +} + +func TestCreateDiscoveryEventEmptyAddress(t *testing.T) { + targets := []target{{ + Address: "", + Name: "routername", + Labels: &[]Label{}, + Operation: "updated"}} + + result, err := createDiscoveryEvent(targets) + if err == nil { + t.Errorf("createDiscoveryEvent(targets) returns %v, want missing address error", result) + } +} + +func TestCreateDiscoveryEventWrongEvent(t *testing.T) { + targets := []target{{ + Address: "1.1.1.1", + Name: "", + Labels: &[]Label{}, + Operation: "wrongOperation"}} + + result, err := createDiscoveryEvent(targets) + if err == nil { + t.Errorf("createDiscoveryEvent(targets) returns %v, want wrong Operation error", result) + } +} + +func TestParseURI(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + router := gin.New() + var result urlStruct + router.POST("/api/v1/:namespace/target-source/:name/createTargets", func(ctx *gin.Context) { + result = parseURI(ctx) + }) + + req := httptest.NewRequest(http.MethodPost, "/api/v1/default/target-source/http-discovery/createTargets", nil) + router.ServeHTTP(recorder, req) + + expected := urlStruct{ + Namespace: "default", + Name: "http-discovery", + } + + if !reflect.DeepEqual(result, expected) { + t.Errorf("parseURI(ctx) = %v; want %v", result, expected) + } + if recorder.Code != http.StatusOK { + t.Errorf("parseURI(ctx) status code = %d; want %d", recorder.Code, http.StatusOK) + } +} + +func TestParseURIMissingName(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + router := gin.New() + var result urlStruct + router.POST("/api/v1/:namespace/target-source/:name/createTargets", func(ctx *gin.Context) { + result = parseURI(ctx) + }) + + req := httptest.NewRequest(http.MethodPost, "/api/v1/default/target-source//createTargets", nil) + router.ServeHTTP(recorder, req) + + if !reflect.DeepEqual(result, urlStruct{}) { + t.Errorf("parseURI(ctx) = %v; want empty urlStruct", result) + } + if recorder.Code != http.StatusBadRequest { + t.Errorf("parseURI(ctx) status code = %d; want %d", recorder.Code, http.StatusBadRequest) + } +}