Skip to content

Feat/huggingface backend #1665

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gaius-qi merged 18 commits into from
Mar 9, 2026
Merged

Feat/huggingface backend #1665

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
6fb5610
feat(backend): add Hugging Face backend module
pmady Feb 7, 2026
0d07910
feat(backend): register Hugging Face backend in BackendFactory
pmady Feb 7, 2026
d73ef0b
chore(backend): add serde dependencies for Hugging Face backend
pmady Feb 7, 2026
6921a90
feat(dfget): add Hugging Face CLI options and examples
pmady Feb 7, 2026
c2ff0ed
style: fix formatting issues
pmady Feb 7, 2026
cdf514f
refactor: address reviewer feedback for huggingface backend
pmady Feb 9, 2026
e3752d1
fix: plumb --hf-token through request headers to HF backend
pmady Feb 9, 2026
49c7fa0
fix: address copilot review comments on HF backend
pmady Feb 9, 2026
b18588c
Merge branch 'main' of github.com:dragonflyoss/client into feat/huggi…
gaius-qi Feb 10, 2026
8cfccf1
feat: refactor Hugging Face backend naming conventions and add Huggin…
gaius-qi Feb 10, 2026
1ede78c
docs(huggingface): update comments for clarity
gaius-qi Feb 14, 2026
0da7f3a
Merge branch 'main' of github.com:dragonflyoss/client into feat/huggi…
gaius-qi Feb 14, 2026
a8debc5
Merge branch 'main' of github.com:dragonflyoss/client into feat/huggi…
gaius-qi Mar 4, 2026
7ade20a
refactor(backend): add hugging face backend with hf:// URL scheme sup…
gaius-qi Mar 4, 2026
d468424
test(hugging_face): remove invalid scheme test
gaius-qi Mar 4, 2026
7ae41cf
fix(error): improve error message for max file limit
gaius-qi Mar 5, 2026
327f057
feat(huggingface): add revision support for API calls
gaius-qi Mar 9, 2026
70f2656
fix(tests): remove revision from parsed URLs
gaius-qi Mar 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.2.1
dragonfly-client-metric = { path = "dragonfly-client-metric", version = "1.2.11" }
dragonfly-client-util = { path = "dragonfly-client-util", version = "1.2.11" }
dragonfly-client-init = { path = "dragonfly-client-init", version = "1.2.11" }
dragonfly-api = "=2.2.14"
dragonfly-api = "=2.2.22"
thiserror = "2.0"
futures = "0.3.31"
reqwest = { version = "0.12.28", features = [
Expand Down
2 changes: 2 additions & 0 deletions dragonfly-client-backend/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ percent-encoding.workspace = true
futures.workspace = true
reqwest-tracing.workspace = true
fastrand.workspace = true
serde.workspace = true
serde_json.workspace = true
dashmap.workspace = true
lru.workspace = true
reqwest-retry = "0.8"
Expand Down
37 changes: 19 additions & 18 deletions dragonfly-client-backend/src/hdfs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@
* limitations under the License.
*/

use crate::{
Backend, Body, DirEntry, ExistsRequest, GetRequest, GetResponse, PutRequest, PutResponse,
StatRequest, StatResponse,
};
use dragonfly_api::common;
use dragonfly_client_core::error::BackendError;
use dragonfly_client_core::{Error as ClientError, Result as ClientResult};
Expand All @@ -33,20 +37,20 @@ const DEFAULT_NAMENODE_PORT: u16 = 9870;
/// Hdfs is a struct that implements the Backend trait.
#[derive(Default)]
pub struct Hdfs {
/// scheme is the scheme of the HDFS.
/// Scheme is the scheme of the HDFS.
scheme: String,
}

/// Hdfs implements the Backend trait.
impl Hdfs {
/// new returns a new HDFS backend.
/// Create a new Hdfs instance.
pub fn new() -> Self {
Self {
scheme: HDFS_SCHEME.to_string(),
}
}

/// operator initializes the operator with the parsed URL and HDFS config.
/// Operator initializes the operator with the parsed URL and HDFS config.
pub fn operator(
&self,
url: Url,
Expand Down Expand Up @@ -82,15 +86,15 @@ impl Hdfs {

/// Implement the Backend trait for Hdfs.
#[tonic::async_trait]
impl super::Backend for Hdfs {
/// scheme returns the scheme of the HDFS backend.
impl Backend for Hdfs {
/// Scheme returns the scheme of the HDFS backend.
fn scheme(&self) -> String {
self.scheme.clone()
}

/// stat gets the metadata from the backend.
/// Stat the metadata from the backend.
#[instrument(skip_all)]
async fn stat(&self, request: super::StatRequest) -> ClientResult<super::StatResponse> {
async fn stat(&self, request: StatRequest) -> ClientResult<StatResponse> {
debug!(
"stat request {} {}: {:?}",
request.task_id, request.url, request.http_header
Expand Down Expand Up @@ -129,7 +133,7 @@ impl super::Backend for Hdfs {
let metadata = entry.metadata();
let mut url = url.clone();
url.set_path(entry.path());
super::DirEntry {
DirEntry {
url: url.to_string(),
content_length: metadata.content_length() as usize,
is_dir: metadata.is_dir(),
Expand Down Expand Up @@ -161,7 +165,7 @@ impl super::Backend for Hdfs {
response.content_length()
);

Ok(super::StatResponse {
Ok(StatResponse {
success: true,
content_length: Some(response.content_length()),
http_header: None,
Expand All @@ -171,12 +175,9 @@ impl super::Backend for Hdfs {
})
}

/// get gets the content from the backend.
/// Get the content from the backend.
#[instrument(skip_all)]
async fn get(
&self,
request: super::GetRequest,
) -> ClientResult<super::GetResponse<super::Body>> {
async fn get(&self, request: GetRequest) -> ClientResult<GetResponse<Body>> {
debug!(
"get request {} {}: {:?}",
request.piece_id, request.url, request.http_header
Expand Down Expand Up @@ -246,15 +247,15 @@ impl super::Backend for Hdfs {
})
}

/// put puts the content to the backend.
/// Put the content to the backend.
#[instrument(skip_all)]
async fn put(&self, _request: super::PutRequest) -> ClientResult<super::PutResponse> {
async fn put(&self, _request: PutRequest) -> ClientResult<PutResponse> {
unimplemented!()
}

/// exists checks whether the file exists in the backend.
/// Exists checks whether the file exists in the backend.
#[instrument(skip_all)]
async fn exists(&self, request: super::ExistsRequest) -> ClientResult<bool> {
async fn exists(&self, request: ExistsRequest) -> ClientResult<bool> {
debug!(
"exist request {} {}: {:?}",
request.task_id, request.url, request.http_header
Expand Down
Loading