From 69246d0837ba376bf78ced0cfb9518caf34d6e28 Mon Sep 17 00:00:00 2001 From: Arsen Gumin Date: Tue, 23 Dec 2025 19:14:27 +0300 Subject: [PATCH 1/2] feature: [WIP] add mlflow support for models://name/version and model/version format --- .golangci.yml | 4 + go.mod | 37 +++++ go.sum | 152 ++++++++++++++++++ pkg/modelprovider/mlflow/downloader.go | 209 +++++++++++++++++++++++++ pkg/modelprovider/mlflow/provider.go | 119 ++++++++++++++ pkg/modelprovider/mlflow/storage.go | 7 + pkg/modelprovider/registry.go | 2 + 7 files changed, 530 insertions(+) create mode 100644 pkg/modelprovider/mlflow/downloader.go create mode 100644 pkg/modelprovider/mlflow/provider.go create mode 100644 pkg/modelprovider/mlflow/storage.go diff --git a/.golangci.yml b/.golangci.yml index 3f621275..4e97d1bd 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -41,6 +41,10 @@ formatters: enable: - gci - gofmt + - gofumpt + - goimports + - golines + - swaggo settings: gci: sections: diff --git a/go.mod b/go.mod index 4d671068..cdc86416 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,11 @@ require ( d7y.io/api/v2 v2.2.1 github.com/antgroup/hugescm v0.18.3 github.com/avast/retry-go/v4 v4.7.0 + github.com/aws/aws-sdk-go-v2 v1.41.0 + github.com/aws/aws-sdk-go-v2/config v1.32.6 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.17 + github.com/aws/aws-sdk-go-v2/service/s3 v1.94.0 + github.com/databricks/databricks-sdk-go v0.95.0 github.com/distribution/distribution/v3 v3.0.0 github.com/distribution/reference v0.6.0 github.com/dragonflyoss/model-spec v0.0.6 @@ -30,12 +35,31 @@ require ( ) require ( + cloud.google.com/go/auth v0.9.3 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect + cloud.google.com/go/compute/metadata v0.7.0 // indirect dario.cat/mergo v1.0.0 // indirect github.com/BurntSushi/toml v1.5.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.3.0 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect + github.com/aws/smithy-go v1.24.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -48,6 +72,7 @@ require ( github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.6.2 // indirect @@ -56,7 +81,10 @@ require ( github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/google/s2a-go v0.1.8 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/gorilla/mux v1.8.2-0.20240619235004-db9d1d0073d2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect @@ -72,6 +100,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect @@ -90,9 +119,11 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/zeebo/blake3 v0.2.4 // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/bridges/prometheus v0.63.0 // indirect go.opentelemetry.io/contrib/exporters/autoexport v0.63.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 // indirect go.opentelemetry.io/otel v1.38.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 // indirect @@ -114,12 +145,18 @@ require ( go.opentelemetry.io/proto/otlp v1.8.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/mod v0.29.0 // indirect golang.org/x/net v0.47.0 // indirect + golang.org/x/oauth2 v0.32.0 // indirect golang.org/x/term v0.37.0 // indirect golang.org/x/text v0.31.0 // indirect + golang.org/x/time v0.6.0 // indirect + google.golang.org/api v0.197.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 0abea2cf..52f0f241 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,15 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go/auth v0.9.3 h1:VOEUIAADkkLtyfr3BLa3R8Ed/j6w1jTBmARx+wb5w5U= +cloud.google.com/go/auth v0.9.3/go.mod h1:7z6VY+7h3KUdRov5F1i8NDP5ZzWKYmEPO842BgCsmTk= +cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= +cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= +cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= +cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= d7y.io/api/v2 v2.2.1 h1:po39adZpbGn6sIl9vR9E1fVbOLim5SOP/3Tt0g8Ii58= d7y.io/api/v2 v2.2.1/go.mod h1:TtW9UE0CebRB/CWIEbWfRnljmpKf/mNoe2qIUO+PoP0= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= @@ -25,20 +33,63 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/avast/retry-go/v4 v4.7.0 h1:yjDs35SlGvKwRNSykujfjdMxMhMQQM0TnIjJaHB+Zio= github.com/avast/retry-go/v4 v4.7.0/go.mod h1:ZMPDa3sY2bKgpLtap9JRUgk2yTAba7cgiFhqxY2Sg6Q= +github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4= +github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= +github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8= +github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.17 h1:fODjlj9c1zIfZYFxdC6Z4GX/plrZUYI/5EklgA/24Hw= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.17/go.mod h1:CEyBu8kavY5Tc8i/8A810DuKydd19Lrx2/TmcNdjOAk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A= +github.com/aws/aws-sdk-go-v2/service/s3 v1.94.0 h1:SWTxh/EcUCDVqi/0s26V6pVUq0BBG7kx0tDTmF/hCgA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.94.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk= +github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= +github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= @@ -46,6 +97,8 @@ github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22r github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0= github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8= +github.com/databricks/databricks-sdk-go v0.95.0 h1:ZehbIygQgK3yeKEJIiichtftwPmiYVb4SrqBukJGbFU= +github.com/databricks/databricks-sdk-go v0.95.0/go.mod h1:hWoHnHbNLjPKiTm5K/7bcIv3J3Pkgo5x9pPzh8K3RVE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -68,8 +121,14 @@ github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4= github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= @@ -98,21 +157,46 @@ github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlnd github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= +github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= github.com/gorilla/mux v1.8.2-0.20240619235004-db9d1d0073d2 h1:oZRjfKe/6Qh676XFYvylkCWd0gu8KVZeZYZwkNw6NAU= github.com/gorilla/mux v1.8.2-0.20240619235004-db9d1d0073d2/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= @@ -179,6 +263,8 @@ github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -192,6 +278,7 @@ github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -235,12 +322,17 @@ github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= @@ -255,12 +347,18 @@ github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI= github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE= github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/bridges/prometheus v0.63.0 h1:/Rij/t18Y7rUayNg7Id6rPrEnHgorxYabm2E6wUdPP4= go.opentelemetry.io/contrib/bridges/prometheus v0.63.0/go.mod h1:AdyDPn6pkbkt2w01n3BubRVk7xAsCRq1Yg1mpfyA/0E= go.opentelemetry.io/contrib/exporters/autoexport v0.63.0 h1:NLnZybb9KkfMXPwZhd5diBYJoVxiO9Qa06dacEA7ySY= go.opentelemetry.io/contrib/exporters/autoexport v0.63.0/go.mod h1:OvRg7gm5WRSCtxzGSsrFHbDLToYlStHNZQ+iPNIyD6g= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 h1:DheMAlT6POBP+gh8RUH19EOTnQIor5QE0uSRPtzCpSw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0/go.mod h1:wZcGmeVO9nzP67aYSLDqXNWK87EZWhi7JWj1v7ZXf94= go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 h1:OMqPldHt79PqWKOMYIAQs3CxAi7RLgPxwfFSwr4ZxtM= @@ -309,33 +407,54 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= +golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= @@ -344,18 +463,47 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.197.0 h1:x6CwqQLsFiA5JKAiGyGBjc2bNtHtLddhJCE2IKuhhcQ= +google.golang.org/api v0.197.0/go.mod h1:AuOuo20GoQ331nq7DquGHlU6d+2wN2fZ8O0ta60nRNw= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda h1:+2XxjfsAu6vqFxwGBRcHiMaDCuZiqXGDUDVWVtrFAnE= google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo= google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0= google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -363,6 +511,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -371,5 +521,7 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= oras.land/oras-go/v2 v2.6.0 h1:X4ELRsiGkrbeox69+9tzTu492FMUu7zJQW6eJU+I2oc= oras.land/oras-go/v2 v2.6.0/go.mod h1:magiQDfG6H1O9APp+rOsvCPcW1GD2MM7vgnKY0Y+u1o= diff --git a/pkg/modelprovider/mlflow/downloader.go b/pkg/modelprovider/mlflow/downloader.go new file mode 100644 index 00000000..5e454d7a --- /dev/null +++ b/pkg/modelprovider/mlflow/downloader.go @@ -0,0 +1,209 @@ +package mlflow + +import ( + "context" + "errors" + "fmt" + "log" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +type MlFlowClient struct { + registry *ml.ModelRegistryAPI +} + +// TODO hdfs +// TODO sql +// TODO file +// Read https://mlflow.org/docs/latest/self-hosting/architecture/backend-store/ +var ( + storageProvider = map[string]StorageBackend{ + "s3": &S3StorageBackend{}, + } +) + +func NewMlFlowRegistry(mlflowClient *client.DatabricksClient) (MlFlowClient, error) { + var registry *ml.ModelRegistryAPI + + if mlflowClient != nil { + registry = ml.NewModelRegistry(mlflowClient) + fmt.Println("Use default mlflow client for MlFlowRegistryAPI") + return MlFlowClient{registry: registry}, nil + } + + //TODO Support more auth methods? + cfg := config.Config{ + //Credentials: config.BasicCredentials{}, + } + mlClient, err := client.New(&cfg) + + if err != nil { + return MlFlowClient{}, err + } + registry = ml.NewModelRegistry(mlClient) + return MlFlowClient{registry: registry}, nil +} + +func (mlfr *MlFlowClient) PullModelByName(ctx context.Context, modelName string, modelVersion string, destSrc string) error { + + if mlfr == nil || mlfr.registry == nil { + return errors.New("Mlflow client is not initialized: registry is nil") + } + + versions, err := mlfr.registry.GetLatestVersionsAll(ctx, ml.GetLatestVersionsRequest{Name: modelName}) + if err != nil { + return errors.Join(errors.New(fmt.Sprintf("failed to get versions for model: %s", modelName)), err) + } + + if len(versions) == 0 { + return errors.New(fmt.Sprintf("model %s has versions: %v", modelName, versions)) + } + + var rawVersion []string + for _, version := range versions { + rawVersion = append(rawVersion, version.Version) + } + contains := slices.Contains(rawVersion, modelVersion) + if !contains { + msg := fmt.Sprintf( + "model %s version %s not found, available version %v", modelName, modelVersion, rawVersion, + ) + return errors.New(msg) + } + + fmt.Printf("Found versions: '%v' for model '%s'\n", rawVersion, modelName) + if modelVersion == "" { + modelVersion = rawVersion[0] + } + + uri, err := mlfr.registry.GetModelVersionDownloadUri(ctx, ml.GetModelVersionDownloadUriRequest{ + Name: modelName, + Version: modelVersion, + }) + + if err != nil { + return err + } + fmt.Printf("Try pull model from uri %s", uri.ArtifactUri) + parsed, err := url.Parse(uri.ArtifactUri) + if parsed == nil { + return errors.New("failed to parse artifact uri") + } + + switch parsed.Scheme { + case "s3": + s3storage := storageProvider[parsed.Scheme] + err = s3storage.DownloadModel(ctx, uri.ArtifactUri+"/", destSrc) // it's dir + if err != nil { + return err + } + default: + msg := fmt.Sprintf("Unsupported artifact storage type: %s", parsed.Scheme) + err = errors.New(msg) + return err + } + + fmt.Printf("✅ Model downloaded") + + return nil +} + +type S3StorageBackend struct { + addressing string +} + +func (s3back *S3StorageBackend) DownloadModel(ctx context.Context, path string, destPath string) error { + + parsed, err := url.Parse(path) + if err != nil { + return err + } + + bucketName := parsed.Host + s3FolderPrefix := parsed.Path[1:] + fmt.Printf("Parsed s3 bucket %s, path %s from path", bucketName, s3FolderPrefix) + if destPath == "" { + destPath = "./downloads/" + } + cfg, err := awsconfig.LoadDefaultConfig(ctx) + if err != nil { + wrap := errors.New(fmt.Sprintf("Error loading AWS config, try change envs or profile: %v\n", err)) + return errors.Join(wrap, err) + } + + log.Printf("Region - %s, endpoint - %s", cfg.Region, aws.ToString(cfg.BaseEndpoint)) + + s3Client := s3.NewFromConfig(cfg) + + var partMiBs int64 = 10 + downloader := manager.NewDownloader(s3Client, func(d *manager.Downloader) { + d.PartSize = partMiBs * 1024 * 1024 + }) + // List objects with the specified prefix + paginator := s3.NewListObjectsV2Paginator(s3Client, &s3.ListObjectsV2Input{ + Bucket: aws.String(bucketName), + Prefix: aws.String(s3FolderPrefix), + Delimiter: aws.String("/"), + }) + + log.Printf("Start downloading from s3 bucket %s, path %s", bucketName, s3FolderPrefix) + + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + log.Printf("Error listing objects: %v\n", err) + return err + } + + for _, object := range page.Contents { + s3Key := *object.Key + log.Printf("Downloading object: %s\n", s3Key) + //if strings.HasSuffix(s3Key, "/") { // Skip S3 "folder" markers + // continue + //} + + // Construct local file path + relativePath := strings.TrimPrefix(s3Key, s3FolderPrefix) + localFilePath := filepath.Join(destPath, relativePath) + + // Create local directories if they don't exist + err = os.MkdirAll(filepath.Dir(localFilePath), 0755) + if err != nil { + log.Printf("Error creating local directory %s: %v\n", filepath.Dir(localFilePath), err) + continue + } + + // Download the object + file, err := os.Create(localFilePath) + if err != nil { + log.Printf("Error creating local file %s: %v\n", localFilePath, err) + continue + } + defer file.Close() + + numBytes, err := downloader.Download(ctx, file, &s3.GetObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(s3Key), + }) + if err != nil { + log.Printf("Error downloading object %s: %v\n", s3Key, err) + continue + } + log.Printf("Downloaded %s to %s (%d bytes)\n", s3Key, localFilePath, numBytes) + } + } + + return nil +} diff --git a/pkg/modelprovider/mlflow/provider.go b/pkg/modelprovider/mlflow/provider.go new file mode 100644 index 00000000..eccfd2b7 --- /dev/null +++ b/pkg/modelprovider/mlflow/provider.go @@ -0,0 +1,119 @@ +/* + * Copyright 2025 The CNAI Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mlflow + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// MlflowProvider implements the modelprovider.Provider interface for Mlflow +type MlflowProvider struct { + mlfClient MlFlowClient +} + +// New creates a new ModelScope provider instance +func New() *MlflowProvider { + return &MlflowProvider{} +} + +// Name returns the name of this provider +func (p *MlflowProvider) Name() string { + return "mlflow" +} + +// SupportsURL checks if this provider can handle the given URL +// Supports experiments url (runs), model-registry(models) +// For short-form repo identifiers (owner/repo), users must explicitly specify --provider mlflow +func (p *MlflowProvider) SupportsURL(url string) bool { + url = strings.TrimSpace(url) + // TODO Mlflow API equals with Databricks Model Registry, support later + possibleUrls := []string{"models", "runs"} + + return hasAnyPrefix(url, possibleUrls) +} + +// DownloadModel downloads a model from ModelScope using the modelscope CLI +func (p *MlflowProvider) DownloadModel(ctx context.Context, modelURL, destDir string) (string, error) { + model, version, err := parseModelURL(modelURL) + xclient := NewMlFlowRegistry(nil) + xclient.PullModelByName(modelURL) + return downloadPath, nil +} + +// CheckAuth verifies that the user is authenticated with MlFlow +func (p *MlflowProvider) CheckAuth() error { + return checkMlflowAuth() +} + +func hasAnyPrefix(s string, subs []string) bool { + for _, sub := range subs { + if strings.HasPrefix(s, sub) { // Check if the main string contains the current substring + return true + } + } + return false +} + +func checkMlflowAuth() error { + var err error + + host := os.Getenv("DATABRICKS_HOST") + usr := os.Getenv("DATABRICKS_USERNAME") + pass := os.Getenv("DATABRICKS_PASSWORD") + mlfhost := os.Getenv("MLFLOW_TRACKING_URI") + mlfuser := os.Getenv("MLFLOW_TRACKING_USERNAME") + mlfpass := os.Getenv("MLFLOW_TRACKING_PASSWORD") + + if host == "" && usr == "" && pass == "" { + fmt.Println("Please set DATABRICKS_HOST environment variable.") + fmt.Println("Please set DATABRICKS_USERNAME environment variable.") + fmt.Println("Please set DATABRICKS_PASSWORD environment variable.") + } else if mlfhost != "" && mlfuser != "" && mlfpass != "" { + err = os.Setenv("DATABRICKS_HOST", mlfhost) + if err != nil { + return err + } + err = os.Setenv("DATABRICKS_USERNAME", usr) + if err != nil { + return err + } + err = os.Setenv("DATABRICKS_PASSWORD", pass) + if err != nil { + return err + } + + } else { + return errors.New("please set MLFLOW tracking environment variable.") + } + return err +} + +func parseModelURL(modelURL string) (name, version, error) { + var err error + var name string + var version string + + name = "" + version = "" + return +} diff --git a/pkg/modelprovider/mlflow/storage.go b/pkg/modelprovider/mlflow/storage.go new file mode 100644 index 00000000..63a11a34 --- /dev/null +++ b/pkg/modelprovider/mlflow/storage.go @@ -0,0 +1,7 @@ +package mlflow + +import "context" + +type StorageBackend interface { + DownloadModel(ctx context.Context, path string, destPath string) error +} diff --git a/pkg/modelprovider/registry.go b/pkg/modelprovider/registry.go index 55de2b69..57b2f76b 100644 --- a/pkg/modelprovider/registry.go +++ b/pkg/modelprovider/registry.go @@ -21,6 +21,7 @@ import ( "sync" "github.com/modelpack/modctl/pkg/modelprovider/huggingface" + "github.com/modelpack/modctl/pkg/modelprovider/mlflow" "github.com/modelpack/modctl/pkg/modelprovider/modelscope" ) @@ -43,6 +44,7 @@ func GetRegistry() *Registry { providers: []Provider{ huggingface.New(), modelscope.New(), + mlflow.New(), // Future providers can be added here: // civitai.New(), }, From f439f83ad510435ae0327983d60fde5861637c22 Mon Sep 17 00:00:00 2001 From: Arsen Gumin Date: Wed, 24 Dec 2025 18:59:16 +0300 Subject: [PATCH 2/2] feat: stable code with tests Signed-off-by: Arsen Gumin --- pkg/modelprovider/mlflow/downloader.go | 64 +++-- pkg/modelprovider/mlflow/downloader_test.go | 116 ++++++++ pkg/modelprovider/mlflow/provider.go | 61 +++- pkg/modelprovider/mlflow/provider_test.go | 295 ++++++++++++++++++++ pkg/modelprovider/registry_test.go | 3 +- 5 files changed, 499 insertions(+), 40 deletions(-) create mode 100644 pkg/modelprovider/mlflow/downloader_test.go create mode 100644 pkg/modelprovider/mlflow/provider_test.go diff --git a/pkg/modelprovider/mlflow/downloader.go b/pkg/modelprovider/mlflow/downloader.go index 5e454d7a..221afd31 100644 --- a/pkg/modelprovider/mlflow/downloader.go +++ b/pkg/modelprovider/mlflow/downloader.go @@ -43,12 +43,11 @@ func NewMlFlowRegistry(mlflowClient *client.DatabricksClient) (MlFlowClient, err return MlFlowClient{registry: registry}, nil } - //TODO Support more auth methods? + // TODO Support more auth methods? cfg := config.Config{ - //Credentials: config.BasicCredentials{}, + // Credentials: config.BasicCredentials{}, } mlClient, err := client.New(&cfg) - if err != nil { return MlFlowClient{}, err } @@ -56,19 +55,26 @@ func NewMlFlowRegistry(mlflowClient *client.DatabricksClient) (MlFlowClient, err return MlFlowClient{registry: registry}, nil } -func (mlfr *MlFlowClient) PullModelByName(ctx context.Context, modelName string, modelVersion string, destSrc string) error { - +func (mlfr *MlFlowClient) PullModelByName( + ctx context.Context, + modelName string, + modelVersion string, + destSrc string, +) (string, error) { if mlfr == nil || mlfr.registry == nil { - return errors.New("Mlflow client is not initialized: registry is nil") + return "", errors.New("mlflow client is not initialized: registry is nil") } - versions, err := mlfr.registry.GetLatestVersionsAll(ctx, ml.GetLatestVersionsRequest{Name: modelName}) + versions, err := mlfr.registry.GetLatestVersionsAll( + ctx, + ml.GetLatestVersionsRequest{Name: modelName}, + ) if err != nil { - return errors.Join(errors.New(fmt.Sprintf("failed to get versions for model: %s", modelName)), err) + return "", errors.Join(fmt.Errorf("failed to get versions for model: %s", modelName), err) } if len(versions) == 0 { - return errors.New(fmt.Sprintf("model %s has versions: %v", modelName, versions)) + return "", fmt.Errorf("model %s has versions: %v", modelName, versions) } var rawVersion []string @@ -78,9 +84,12 @@ func (mlfr *MlFlowClient) PullModelByName(ctx context.Context, modelName string, contains := slices.Contains(rawVersion, modelVersion) if !contains { msg := fmt.Sprintf( - "model %s version %s not found, available version %v", modelName, modelVersion, rawVersion, + "model %s version %s not found, available version %v", + modelName, + modelVersion, + rawVersion, ) - return errors.New(msg) + return "", errors.New(msg) } fmt.Printf("Found versions: '%v' for model '%s'\n", rawVersion, modelName) @@ -92,40 +101,43 @@ func (mlfr *MlFlowClient) PullModelByName(ctx context.Context, modelName string, Name: modelName, Version: modelVersion, }) - if err != nil { - return err + return "", err } fmt.Printf("Try pull model from uri %s", uri.ArtifactUri) parsed, err := url.Parse(uri.ArtifactUri) if parsed == nil { - return errors.New("failed to parse artifact uri") + return "", errors.New("failed to parse artifact uri") } switch parsed.Scheme { case "s3": s3storage := storageProvider[parsed.Scheme] + destSrc = filepath.Join(destSrc, modelName) err = s3storage.DownloadModel(ctx, uri.ArtifactUri+"/", destSrc) // it's dir if err != nil { - return err + return "", err } default: msg := fmt.Sprintf("Unsupported artifact storage type: %s", parsed.Scheme) err = errors.New(msg) - return err + return "", err } fmt.Printf("✅ Model downloaded") - return nil + return destSrc, nil } type S3StorageBackend struct { addressing string } -func (s3back *S3StorageBackend) DownloadModel(ctx context.Context, path string, destPath string) error { - +func (s3back *S3StorageBackend) DownloadModel( + ctx context.Context, + path string, + destPath string, +) error { parsed, err := url.Parse(path) if err != nil { return err @@ -134,12 +146,10 @@ func (s3back *S3StorageBackend) DownloadModel(ctx context.Context, path string, bucketName := parsed.Host s3FolderPrefix := parsed.Path[1:] fmt.Printf("Parsed s3 bucket %s, path %s from path", bucketName, s3FolderPrefix) - if destPath == "" { - destPath = "./downloads/" - } + cfg, err := awsconfig.LoadDefaultConfig(ctx) if err != nil { - wrap := errors.New(fmt.Sprintf("Error loading AWS config, try change envs or profile: %v\n", err)) + wrap := fmt.Errorf("Error loading AWS config, try change envs or profile: %v\n", err) return errors.Join(wrap, err) } @@ -179,9 +189,13 @@ func (s3back *S3StorageBackend) DownloadModel(ctx context.Context, path string, localFilePath := filepath.Join(destPath, relativePath) // Create local directories if they don't exist - err = os.MkdirAll(filepath.Dir(localFilePath), 0755) + err = os.MkdirAll(filepath.Dir(localFilePath), 0o755) if err != nil { - log.Printf("Error creating local directory %s: %v\n", filepath.Dir(localFilePath), err) + log.Printf( + "Error creating local directory %s: %v\n", + filepath.Dir(localFilePath), + err, + ) continue } diff --git a/pkg/modelprovider/mlflow/downloader_test.go b/pkg/modelprovider/mlflow/downloader_test.go new file mode 100644 index 00000000..b5382f03 --- /dev/null +++ b/pkg/modelprovider/mlflow/downloader_test.go @@ -0,0 +1,116 @@ +package mlflow + +import ( + "context" + "testing" + + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +func TestMlFlowClient_PullModelByName(t *testing.T) { + type fields struct { + registry *ml.ModelRegistryAPI + } + type args struct { + ctx context.Context + modelName string + modelVersion string + destSrc string + } + tests := []struct { + name string + fields fields + args args + want string + wantErr bool + }{ + { + name: "nil receiver returns error", + fields: fields{registry: nil}, + args: args{ctx: context.Background(), modelName: "model", modelVersion: "1", destSrc: "/tmp"}, + want: "", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mlfr := &MlFlowClient{ + registry: tt.fields.registry, + } + got, err := mlfr.PullModelByName(tt.args.ctx, tt.args.modelName, tt.args.modelVersion, tt.args.destSrc) + if (err != nil) != tt.wantErr { + t.Errorf("PullModelByName() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("PullModelByName() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNewMlFlowRegistry(t *testing.T) { + type args struct { + mlflowClient *client.DatabricksClient + } + tests := []struct { + name string + args args + want MlFlowClient + wantErr bool + }{ + { + name: "non-nil client returns registry", + args: args{mlflowClient: &client.DatabricksClient{}}, + want: MlFlowClient{}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewMlFlowRegistry(tt.args.mlflowClient) + if (err != nil) != tt.wantErr { + t.Errorf("NewMlFlowRegistry() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr && got.registry == nil { + t.Errorf("NewMlFlowRegistry() registry is nil") + } + }) + } +} + +func TestS3StorageBackend_DownloadModel(t *testing.T) { + type fields struct { + addressing string + } + type args struct { + ctx context.Context + path string + destPath string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "invalid url returns error", + fields: fields{addressing: ""}, + args: args{ctx: context.Background(), path: "http://[::1", destPath: "/tmp"}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s3back := &S3StorageBackend{ + addressing: tt.fields.addressing, + } + if err := s3back.DownloadModel(tt.args.ctx, tt.args.path, tt.args.destPath); (err != nil) != tt.wantErr { + t.Errorf("DownloadModel() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/modelprovider/mlflow/provider.go b/pkg/modelprovider/mlflow/provider.go index eccfd2b7..3207685c 100644 --- a/pkg/modelprovider/mlflow/provider.go +++ b/pkg/modelprovider/mlflow/provider.go @@ -20,15 +20,14 @@ import ( "context" "errors" "fmt" + "net/url" "os" - "os/exec" - "path/filepath" "strings" ) // MlflowProvider implements the modelprovider.Provider interface for Mlflow type MlflowProvider struct { - mlfClient MlFlowClient + mflClient MlFlowClient } // New creates a new ModelScope provider instance @@ -47,7 +46,7 @@ func (p *MlflowProvider) Name() string { func (p *MlflowProvider) SupportsURL(url string) bool { url = strings.TrimSpace(url) // TODO Mlflow API equals with Databricks Model Registry, support later - possibleUrls := []string{"models", "runs"} + possibleUrls := []string{"models"} return hasAnyPrefix(url, possibleUrls) } @@ -55,8 +54,17 @@ func (p *MlflowProvider) SupportsURL(url string) bool { // DownloadModel downloads a model from ModelScope using the modelscope CLI func (p *MlflowProvider) DownloadModel(ctx context.Context, modelURL, destDir string) (string, error) { model, version, err := parseModelURL(modelURL) - xclient := NewMlFlowRegistry(nil) - xclient.PullModelByName(modelURL) + if err != nil { + return "", err + } + registryClient, err := NewMlFlowRegistry(nil) + if err != nil { + return "", err + } + downloadPath, err := registryClient.PullModelByName(ctx, model, version, destDir) + if err != nil { + return "", err + } return downloadPath, nil } @@ -75,6 +83,7 @@ func hasAnyPrefix(s string, subs []string) bool { } func checkMlflowAuth() error { + var err error host := os.Getenv("DATABRICKS_HOST") @@ -88,7 +97,10 @@ func checkMlflowAuth() error { fmt.Println("Please set DATABRICKS_HOST environment variable.") fmt.Println("Please set DATABRICKS_USERNAME environment variable.") fmt.Println("Please set DATABRICKS_PASSWORD environment variable.") - } else if mlfhost != "" && mlfuser != "" && mlfpass != "" { + } else { + return nil + } + if mlfhost != "" && mlfuser != "" && mlfpass != "" { err = os.Setenv("DATABRICKS_HOST", mlfhost) if err != nil { return err @@ -108,12 +120,33 @@ func checkMlflowAuth() error { return err } -func parseModelURL(modelURL string) (name, version, error) { - var err error - var name string - var version string +func parseModelURL(modelURL string) (string, string, error) { + if modelURL == "" { + return "", "", errors.New("modelUrl value missing.") + } + + if strings.HasPrefix(modelURL, "models:") { + parse, err := url.Parse(modelURL) + if err != nil { + return "", "", err + } + + if parse == nil { + return "", "", errors.New("model url is nil") + } + + return parse.Hostname(), strings.TrimLeft(parse.Path, "/"), nil + + } else if strings.Contains(modelURL, "/") { - name = "" - version = "" - return + split := strings.Split(modelURL, "/") + + if len(split) != 2 { + return "", "", errors.New("model url is invalid, valid mask name/version") + } + return split[0], split[1], nil + + } else { + return modelURL, "", nil + } } diff --git a/pkg/modelprovider/mlflow/provider_test.go b/pkg/modelprovider/mlflow/provider_test.go new file mode 100644 index 00000000..27810798 --- /dev/null +++ b/pkg/modelprovider/mlflow/provider_test.go @@ -0,0 +1,295 @@ +package mlflow + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMlflowProvider_CheckAuth(t *testing.T) { + type fields struct { + mlfClient MlFlowClient + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "missing envs returns error", + wantErr: true, + }, + { + name: "databricks host set returns nil", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("DATABRICKS_HOST", "") + t.Setenv("DATABRICKS_USERNAME", "") + t.Setenv("DATABRICKS_PASSWORD", "") + t.Setenv("MLFLOW_TRACKING_URI", "") + t.Setenv("MLFLOW_TRACKING_USERNAME", "") + t.Setenv("MLFLOW_TRACKING_PASSWORD", "") + if tt.name == "databricks host set returns nil" { + t.Setenv("DATABRICKS_HOST", "https://example.com") + } + + p := &MlflowProvider{ + mflClient: tt.fields.mlfClient, + } + if err := p.CheckAuth(); (err != nil) != tt.wantErr { + t.Errorf("CheckAuth() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestMlflowProvider_DownloadModel(t *testing.T) { + type fields struct { + mlfClient MlFlowClient + } + type args struct { + ctx context.Context + modelURL string + destDir string + } + tests := []struct { + name string + fields fields + args args + want string + wantErr bool + }{ + { + name: "empty model url returns error", + args: args{ctx: context.Background(), modelURL: "", destDir: "/tmp"}, + want: "", + wantErr: true, + }, + { + name: "invalid model url returns error", + args: args{ + ctx: context.Background(), + modelURL: "http://my-model/1", + destDir: "/tmp", + }, + want: "", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &MlflowProvider{ + mflClient: tt.fields.mlfClient, + } + got, err := p.DownloadModel(tt.args.ctx, tt.args.modelURL, tt.args.destDir) + if (err != nil) != tt.wantErr { + t.Errorf("DownloadModel() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("DownloadModel() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestMlflowProvider_Name(t *testing.T) { + type fields struct { + mlfClient MlFlowClient + } + tests := []struct { + name string + fields fields + want string + }{ + { + name: "returns provider name", + want: "mlflow", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &MlflowProvider{ + mflClient: tt.fields.mlfClient, + } + if got := p.Name(); got != tt.want { + t.Errorf("Name() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestMlflowProvider_SupportsURL(t *testing.T) { + type fields struct { + mlfClient MlFlowClient + } + type args struct { + url string + } + tests := []struct { + name string + fields fields + args args + want bool + }{ + { + name: "models scheme supported", + args: args{url: "models://my-model/1"}, + want: true, + }, + { + name: "trimmed whitespace supported", + args: args{url: " models://my-model/1 "}, + want: true, + }, + { + name: "http scheme not supported", + args: args{url: "http://my-model/1"}, + want: false, + }, + { + name: "empty url not supported", + args: args{url: ""}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &MlflowProvider{ + mflClient: tt.fields.mlfClient, + } + if got := p.SupportsURL(tt.args.url); got != tt.want { + t.Errorf("SupportsURL() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_checkMlflowAuth(t *testing.T) { + tests := []struct { + name string + wantErr bool + }{ + { + name: "missing databricks and mlflow envs returns error", + wantErr: true, + }, + { + name: "databricks host set returns nil", + wantErr: false, + }, + { + name: "mlflow tracking set returns nil", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("DATABRICKS_HOST", "") + t.Setenv("DATABRICKS_USERNAME", "") + t.Setenv("DATABRICKS_PASSWORD", "") + t.Setenv("MLFLOW_TRACKING_URI", "") + t.Setenv("MLFLOW_TRACKING_USERNAME", "") + t.Setenv("MLFLOW_TRACKING_PASSWORD", "") + switch tt.name { + case "databricks host set returns nil": + t.Setenv("DATABRICKS_HOST", "https://example.com") + case "mlflow tracking set returns nil": + t.Setenv("MLFLOW_TRACKING_URI", "https://mlflow.example.com") + t.Setenv("MLFLOW_TRACKING_USERNAME", "mlf-user") + t.Setenv("MLFLOW_TRACKING_PASSWORD", "mlf-pass") + } + + err := checkMlflowAuth() + assert.Equal(t, tt.wantErr, err != nil) + }) + } +} + +func Test_hasAnyPrefix(t *testing.T) { + type args struct { + s string + subs []string + } + tests := []struct { + name string + args args + want bool + }{ + { + "One valid and one unvalid", + args{"models://my-model/1", []string{"models", "invalid"}}, + true, + }, + {"All unvalid", args{"http://my-model/1", []string{"models", "invalid"}}, false}, + {"Empty substrings", args{"models://my-model/1", []string{}}, false}, + {"Empty main string", args{"", []string{"models", "invalid"}}, false}, + {"Both empty", args{"", []string{}}, false}, + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := hasAnyPrefix(tt.args.s, tt.args.subs); got != tt.want { + assert.Equal(t, tt.want, got) + } + }) + } +} + +func Test_parseModelURL(t *testing.T) { + type args struct { + modelURL string + } + tests := []struct { + name string + args args + want string + want1 string + wantErr bool + }{ + // TODO: Add test cases. + { + "models name with mlflow schema", + args{modelURL: "models://my-model/1"}, + "my-model", + "1", + false, + }, + {"models name with http schema", args{modelURL: "http://my-model/1"}, "", "", true}, + {"models name without version", args{modelURL: "my-model"}, "my-model", "", false}, + { + "models with schema and without version", + args{modelURL: "models://my-model"}, + "my-model", + "", + false, + }, + {"invalid url", args{modelURL: "://my-model/1"}, "", "", true}, + { + "model without schema should return error", + args{modelURL: "my-model/1"}, + "my-model", + "1", + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1, err := parseModelURL(tt.args.modelURL) + assert.Equal( + t, + tt.wantErr, + err != nil, + "parseModelURL() error = %v, wantErr %v", + err, + tt.wantErr, + ) + assert.Equal(t, tt.want, got, "parseModelURL() got = %v, want %v", got, tt.want) + assert.Equal(t, tt.want1, got1, "parseModelURL() got1 = %v, want %v", got1, tt.want1) + }) + } +} diff --git a/pkg/modelprovider/registry_test.go b/pkg/modelprovider/registry_test.go index 23d5ad46..1e3f80b6 100644 --- a/pkg/modelprovider/registry_test.go +++ b/pkg/modelprovider/registry_test.go @@ -142,13 +142,14 @@ func TestRegistry_ListProviders(t *testing.T) { registry := GetRegistry() providers := registry.ListProviders() - if len(providers) != 2 { + if len(providers) != 3 { t.Errorf("ListProviders() returned %d providers, want 2", len(providers)) } expectedProviders := map[string]bool{ "huggingface": false, "modelscope": false, + "mlflow": false, } for _, name := range providers {