Skip to content

feat: add ModelScope backend support with modelscope:// protocol #1673

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gaius-qi merged 4 commits into from
Mar 10, 2026
Merged

feat: add ModelScope backend support with modelscope:// protocol #1673

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9597b39
feat: add ModelScope backend support with modelscope:// protocol
pmady Feb 9, 2026
254536c
Merge branch 'main' of github.com:dragonflyoss/client into feat/model…
gaius-qi Mar 9, 2026
cc744bf
test(model_scope): update tests for download URL changes
gaius-qi Mar 9, 2026
425c2c0
fix(http): improve error handling for HTTP responses
gaius-qi Mar 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.2.1
dragonfly-client-metric = { path = "dragonfly-client-metric", version = "1.2.11" }
dragonfly-client-util = { path = "dragonfly-client-util", version = "1.2.11" }
dragonfly-client-init = { path = "dragonfly-client-init", version = "1.2.11" }
dragonfly-api = "=2.2.23"
dragonfly-api = "=2.2.24"
thiserror = "2.0"
futures = "0.3.31"
reqwest = { version = "0.12.28", features = [
Expand Down
4 changes: 2 additions & 2 deletions dragonfly-client-backend/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ tracing.workspace = true
opendal.workspace = true
percent-encoding.workspace = true
futures.workspace = true
reqwest-tracing.workspace = true
fastrand.workspace = true
serde.workspace = true
serde_json.workspace = true
reqwest-tracing.workspace = true
fastrand.workspace = true
dashmap.workspace = true
lru.workspace = true
reqwest-retry = "0.8"
Expand Down
24 changes: 21 additions & 3 deletions dragonfly-client-backend/src/http.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,7 @@ impl Backend for HTTP {
http_header: None,
http_status_code: None,
entries: Vec::new(),
error_message: None,
error_message: Some(err.to_string()),
});
}
}
Expand All @@ -437,7 +437,9 @@ impl Backend for HTTP {
http_header: None,
http_status_code: None,
entries: Vec::new(),
error_message: None,
error_message: Some(
"got 307 Temporary Redirect without Location header".to_string(),
),
});
}
}
Expand Down Expand Up @@ -473,7 +475,7 @@ impl Backend for HTTP {
http_header: None,
http_status_code: None,
entries: Vec::new(),
error_message: None,
error_message: Some(err.to_string()),
});
}
}
Expand Down Expand Up @@ -903,6 +905,7 @@ LJ8gCHKBOJy9dW62DcRWw6zzlTtt9y18/Btx0Hpawg==
object_storage: None,
hdfs: None,
hugging_face: None,
model_scope: None,
})
.await
.unwrap();
Expand Down Expand Up @@ -933,6 +936,7 @@ LJ8gCHKBOJy9dW62DcRWw6zzlTtt9y18/Btx0Hpawg==
object_storage: None,
hdfs: None,
hugging_face: None,
model_scope: None,
})
.await;

Expand Down Expand Up @@ -965,6 +969,7 @@ LJ8gCHKBOJy9dW62DcRWw6zzlTtt9y18/Btx0Hpawg==
object_storage: None,
hdfs: None,
hugging_face: None,
model_scope: None,
})
.await
.unwrap();
Expand All @@ -987,6 +992,7 @@ LJ8gCHKBOJy9dW62DcRWw6zzlTtt9y18/Btx0Hpawg==
object_storage: None,
hdfs: None,
hugging_face: None,
model_scope: None,
})
.await
.unwrap();
Expand All @@ -1008,6 +1014,7 @@ LJ8gCHKBOJy9dW62DcRWw6zzlTtt9y18/Btx0Hpawg==
object_storage: None,
hdfs: None,
hugging_face: None,
model_scope: None,
})
.await;

Expand All @@ -1030,6 +1037,7 @@ LJ8gCHKBOJy9dW62DcRWw6zzlTtt9y18/Btx0Hpawg==
object_storage: None,
hdfs: None,
hugging_face: None,
model_scope: None,
})
.await
.unwrap();
Expand All @@ -1054,6 +1062,7 @@ LJ8gCHKBOJy9dW62DcRWw6zzlTtt9y18/Btx0Hpawg==
object_storage: None,
hdfs: None,
hugging_face: None,
model_scope: None,
})
.await;

Expand All @@ -1074,6 +1083,7 @@ LJ8gCHKBOJy9dW62DcRWw6zzlTtt9y18/Btx0Hpawg==
object_storage: None,
hdfs: None,
hugging_face: None,
model_scope: None,
})
.await
.unwrap();
Expand All @@ -1098,6 +1108,7 @@ LJ8gCHKBOJy9dW62DcRWw6zzlTtt9y18/Btx0Hpawg==
object_storage: None,
hdfs: None,
hugging_face: None,
model_scope: None,
})
.await
.unwrap();
Expand Down Expand Up @@ -1129,6 +1140,7 @@ LJ8gCHKBOJy9dW62DcRWw6zzlTtt9y18/Btx0Hpawg==
object_storage: None,
hdfs: None,
hugging_face: None,
model_scope: None,
})
.await
.unwrap();
Expand Down Expand Up @@ -1159,6 +1171,7 @@ LJ8gCHKBOJy9dW62DcRWw6zzlTtt9y18/Btx0Hpawg==
object_storage: None,
hdfs: None,
hugging_face: None,
model_scope: None,
})
.await
.unwrap();
Expand Down Expand Up @@ -1189,6 +1202,7 @@ LJ8gCHKBOJy9dW62DcRWw6zzlTtt9y18/Btx0Hpawg==
object_storage: None,
hdfs: None,
hugging_face: None,
model_scope: None,
})
.await;

Expand Down Expand Up @@ -1345,6 +1359,7 @@ LJ8gCHKBOJy9dW62DcRWw6zzlTtt9y18/Btx0Hpawg==
object_storage: None,
hdfs: None,
hugging_face: None,
model_scope: None,
})
.await
.unwrap();
Expand All @@ -1365,6 +1380,7 @@ LJ8gCHKBOJy9dW62DcRWw6zzlTtt9y18/Btx0Hpawg==
object_storage: None,
hdfs: None,
hugging_face: None,
model_scope: None,
})
.await
.unwrap();
Expand Down Expand Up @@ -1411,6 +1427,7 @@ LJ8gCHKBOJy9dW62DcRWw6zzlTtt9y18/Btx0Hpawg==
object_storage: None,
hdfs: None,
hugging_face: None,
model_scope: None,
})
.await
.unwrap();
Expand All @@ -1433,6 +1450,7 @@ LJ8gCHKBOJy9dW62DcRWw6zzlTtt9y18/Btx0Hpawg==
object_storage: None,
hdfs: None,
hugging_face: None,
model_scope: None,
})
.await
.unwrap();
Expand Down
19 changes: 5 additions & 14 deletions dragonfly-client-backend/src/hugging_face.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ struct Lfs {

/// A parsed representation of a Hugging Face URL.
///
/// Format: `hf://[<repository_type>/]<owner>/<repository>[/<path>][@<revision>]`
/// Format: `hf://[<repository_type>/]<owner>/<repository>[/<path>]`
#[derive(Debug, Clone)]
pub struct ParsedURL {
/// The original, unparsed URL.
Expand Down Expand Up @@ -140,20 +140,10 @@ impl RepositoryType {

/// Parses a Hugging Face URL into its constituent components.
///
/// URL Format: hf://[<repository_type>/]<owner>/<repository>[/<path>][@<revision>]
/// URL Format: hf://[<repository_type>/]<owner>/<repository>[/<path>]
/// - repository_type Optional. One of "models" (default), "datasets", or "spaces".
/// - owner/repository Required. For example, "meta-llama/Llama-2-7b".
/// - path Optional file path within the repository.
/// - revision Optional git ref after "@".
///
/// Examples:
/// - URL: https://huggingface.co/meta-llama/Llama-2-7b => Type: Model, Repository ID: meta-llama/Llama-2-7b, Path: None, Revision: main
/// - URL: https://huggingface.co/datasets/squad/data/train.json@v1.0 => Type: Dataset, Repository ID: squad/data, Path: train.json, Revision: v1.0
/// - URL: https://huggingface.co/models/owner/repo/path/to/file@dev => Type: Model, Repository ID: owner/repo, Path: path/to/file, Revision: dev
///
/// Errors:
/// Returns Error::InvalidURI if the URL has no host, or Error::InvalidParameter
/// if the path does not contain at least an owner and repository name.
impl TryFrom<Url> for ParsedURL {
type Error = Error;

Expand Down Expand Up @@ -399,7 +389,7 @@ impl Backend for HuggingFace {
http_header: None,
http_status_code: None,
entries: Vec::new(),
error_message: None,
error_message: Some(err.to_string()),
});
}
};
Expand Down Expand Up @@ -453,7 +443,7 @@ impl Backend for HuggingFace {
http_header: None,
http_status_code: None,
entries: Vec::new(),
error_message: None,
error_message: Some(err.to_string()),
});
}
};
Expand Down Expand Up @@ -586,6 +576,7 @@ impl Backend for HuggingFace {

return Err(Error::InvalidParameter);
};

let download_url = Self::build_download_url(&parsed_url, file_path, &revision);
let response = match self
.client
Expand Down
35 changes: 32 additions & 3 deletions dragonfly-client-backend/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* limitations under the License.
*/

use dragonfly_api::common::v2::{Hdfs, HuggingFace, ObjectStorage, Range};
use dragonfly_api::common::v2::{Hdfs, HuggingFace, ModelScope, ObjectStorage, Range};
use dragonfly_client_config::dfdaemon::Config;
use dragonfly_client_core::{
error::{ErrorType, OrErr},
Expand All @@ -35,6 +35,7 @@ use url::Url;
pub mod hdfs;
pub mod http;
pub mod hugging_face;
pub mod model_scope;
pub mod object_storage;

/// POOL_MAX_IDLE_PER_HOST is the max idle connections per host.
Expand Down Expand Up @@ -92,6 +93,9 @@ pub struct StatRequest {

/// Hugging Face is the hugging face related information.
pub hugging_face: Option<HuggingFace>,

/// Model Scope is the model scope related information.
pub model_scope: Option<ModelScope>,
}

/// StatResponse is the stat response for backend.
Expand Down Expand Up @@ -148,6 +152,9 @@ pub struct GetRequest {

/// Hugging Face is the hugging face related information.
pub hugging_face: Option<HuggingFace>,

/// Model Scope is the model scope related information.
pub model_scope: Option<ModelScope>,
}

/// GetResponse is the get response for backend.
Expand Down Expand Up @@ -223,6 +230,9 @@ pub struct ExistsRequest {

/// Hugging Face is the hugging face related information.
pub hugging_face: Option<HuggingFace>,

/// Model Scope is the model scope related information.
pub model_scope: Option<ModelScope>,
}

/// PutRequest is the put request for backend.
Expand Down Expand Up @@ -253,6 +263,9 @@ pub struct PutRequest {

/// Hugging Face is the hugging face related information.
pub hugging_face: Option<HuggingFace>,

/// Model Scope is the model scope related information.
pub model_scope: Option<ModelScope>,
}

/// PutResponse is the put response for backend.
Expand Down Expand Up @@ -461,6 +474,12 @@ impl BackendFactory {
.insert("hdfs".to_string(), Box::new(hdfs::Hdfs::new()));
info!("load [hdfs] builtin backend");

self.backends.insert(
model_scope::MODEL_SCOPE_SCHEME.to_string(),
Box::new(model_scope::ModelScope::new(self.config.clone())?),
);
info!("load [modelscope] builtin backend");

self.backends.insert(
"hf".to_string(),
Box::new(hugging_face::HuggingFace::new(self.config.clone())?),
Expand Down Expand Up @@ -526,7 +545,17 @@ mod tests {
fn should_load_builtin_backends() {
let factory = BackendFactory::new(Arc::new(Config::default()), None).unwrap();
let expected_backends = vec![
"http", "https", "s3", "gs", "abs", "oss", "obs", "cos", "hdfs", "hf",
"http",
"https",
"s3",
"gs",
"abs",
"oss",
"obs",
"cos",
"hdfs",
"hf",
"modelscope",
];
for backend in expected_backends {
assert!(factory.backends.contains_key(backend));
Expand Down Expand Up @@ -558,7 +587,7 @@ mod tests {
let plugin_dir = dir.path().join("non_existent_plugin_dir");

let factory = BackendFactory::new(Arc::new(Config::default()), Some(&plugin_dir)).unwrap();
assert_eq!(factory.backends.len(), 10);
assert_eq!(factory.backends.len(), 11);
}

#[test]
Expand Down
Loading