From 39de3abb243d1bc74e4859e9faaa23ea2eb0c380 Mon Sep 17 00:00:00 2001 From: yokinanya Date: Fri, 3 Apr 2026 13:59:20 +0800 Subject: [PATCH] feat(drivers/weiyun_open): support weiyun official mcp api --- drivers/all.go | 1 + drivers/weiyun_open/client.go | 131 ++++++++++++ drivers/weiyun_open/driver.go | 234 ++++++++++++++++++++ drivers/weiyun_open/hash.go | 248 ++++++++++++++++++++++ drivers/weiyun_open/hash_test.go | 25 +++ drivers/weiyun_open/json_number.go | 72 +++++++ drivers/weiyun_open/json_number_test.go | 62 ++++++ drivers/weiyun_open/meta.go | 29 +++ drivers/weiyun_open/types.go | 233 ++++++++++++++++++++ drivers/weiyun_open/upload.go | 140 ++++++++++++ drivers/weiyun_open/upload_result.go | 120 +++++++++++ drivers/weiyun_open/upload_result_test.go | 26 +++ 12 files changed, 1321 insertions(+) create mode 100644 drivers/weiyun_open/client.go create mode 100644 drivers/weiyun_open/driver.go create mode 100644 drivers/weiyun_open/hash.go create mode 100644 drivers/weiyun_open/hash_test.go create mode 100644 drivers/weiyun_open/json_number.go create mode 100644 drivers/weiyun_open/json_number_test.go create mode 100644 drivers/weiyun_open/meta.go create mode 100644 drivers/weiyun_open/types.go create mode 100644 drivers/weiyun_open/upload.go create mode 100644 drivers/weiyun_open/upload_result.go create mode 100644 drivers/weiyun_open/upload_result_test.go diff --git a/drivers/all.go b/drivers/all.go index fb68d0395..9cdbbbdd7 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -79,6 +79,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/virtual" _ "github.com/OpenListTeam/OpenList/v4/drivers/webdav" _ "github.com/OpenListTeam/OpenList/v4/drivers/weiyun" + _ "github.com/OpenListTeam/OpenList/v4/drivers/weiyun_open" _ "github.com/OpenListTeam/OpenList/v4/drivers/wopan" _ "github.com/OpenListTeam/OpenList/v4/drivers/wps" _ "github.com/OpenListTeam/OpenList/v4/drivers/yandex_disk" diff --git a/drivers/weiyun_open/client.go b/drivers/weiyun_open/client.go new file mode 100644 index 000000000..32494b2bb --- /dev/null +++ b/drivers/weiyun_open/client.go @@ -0,0 +1,131 @@ +package weiyun_open + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/go-resty/resty/v2" +) + +const ( + jsonRPCVersion = "2.0" + toolCallMethod = "tools/call" +) + +type mcpClient struct { + apiURL string + envID string + token string + http *resty.Client +} + +type rpcRequest struct { + Version string `json:"jsonrpc"` + ID int64 `json:"id"` + Method string `json:"method"` + Params rpcParams `json:"params"` +} + +type rpcParams struct { + Name string `json:"name"` + Arguments any `json:"arguments"` +} + +type rpcResponse struct { + Error *rpcError `json:"error"` + Result rpcResult `json:"result"` +} + +type rpcError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +type rpcResult struct { + Content []rpcContent `json:"content"` +} + +type rpcContent struct { + Type string `json:"type"` + Text string `json:"text"` +} + +func newMCPClient(addition Addition) *mcpClient { + apiURL := addition.APIURL + if apiURL == "" { + apiURL = defaultAPIURL + } + return &mcpClient{ + apiURL: apiURL, + envID: addition.EnvID, + token: addition.MCPToken, + http: base.NewRestyClient(), + } +} + +func (c *mcpClient) call(ctx context.Context, name string, args any, out any) error { + reqBody := rpcRequest{ + Version: jsonRPCVersion, + ID: time.Now().UnixNano(), + Method: toolCallMethod, + Params: rpcParams{ + Name: name, + Arguments: args, + }, + } + resp, err := c.http.R(). + SetContext(ctx). + SetHeaders(c.headers()). + SetBody(&reqBody). + Post(c.apiURL) + if err != nil { + return err + } + if resp.IsError() { + return fmt.Errorf("weiyun mcp http %d: %s", resp.StatusCode(), trimBody(resp.String())) + } + var rpcResp rpcResponse + if err = json.Unmarshal(resp.Body(), &rpcResp); err != nil { + return err + } + if rpcResp.Error != nil { + return fmt.Errorf("weiyun mcp rpc error %d: %s", rpcResp.Error.Code, rpcResp.Error.Message) + } + text, err := rpcResp.Result.text() + if err != nil { + return err + } + return json.Unmarshal([]byte(text), out) +} + +func (c *mcpClient) headers() map[string]string { + headers := map[string]string{ + "Content-Type": "application/json", + "WyHeader": "mcp_token=" + c.token, + } + if c.envID != "" { + headers["Cookie"] = "env_id=" + c.envID + } + return headers +} + +func (r rpcResult) text() (string, error) { + for _, item := range r.Content { + if item.Type == "text" { + return item.Text, nil + } + } + return "", fmt.Errorf("weiyun mcp response missing text content") +} + +func trimBody(body string) string { + body = strings.TrimSpace(body) + if len(body) <= 200 { + return body + } + return body[:200] +} diff --git a/drivers/weiyun_open/driver.go b/drivers/weiyun_open/driver.go new file mode 100644 index 000000000..3ec75cc42 --- /dev/null +++ b/drivers/weiyun_open/driver.go @@ -0,0 +1,234 @@ +package weiyun_open + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +type WeiYunOpen struct { + model.Storage + Addition + + client *mcpClient + root *Folder +} + +func (d *WeiYunOpen) Config() driver.Config { + return config +} + +func (d *WeiYunOpen) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *WeiYunOpen) Init(ctx context.Context) error { + if d.MCPToken == "" { + return errs.EmptyToken + } + if d.RootDirKey != "" && d.RootPDirKey == "" { + return errors.New("root_pdir_key is required when root_dir_key is set") + } + d.client = newMCPClient(d.Addition) + root, err := d.discoverRoot(ctx) + if err != nil { + return err + } + d.root = root + return nil +} + +func (d *WeiYunOpen) Drop(ctx context.Context) error { + d.client = nil + d.root = nil + return nil +} + +func (d *WeiYunOpen) GetRoot(ctx context.Context) (model.Obj, error) { + if d.root == nil { + return nil, errors.New("weiyun open driver is not initialized") + } + return d.root, nil +} + +func (d *WeiYunOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + folder, ok := dir.(*Folder) + if !ok { + return nil, errs.NotSupport + } + offset := 0 + objects := make([]model.Obj, 0) + for { + page, err := d.listPage(ctx, folder, offset) + if err != nil { + return nil, err + } + objects = append(objects, d.pageObjects(page)...) + if page.FinishFlag { + return objects, nil + } + pageCount := len(page.DirList) + len(page.FileList) + if pageCount == 0 { + return nil, errors.New("weiyun list returned empty page before finish") + } + offset += pageCount + } +} + +func (d *WeiYunOpen) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + target, ok := file.(*File) + if !ok { + return nil, errs.NotSupport + } + resp := downloadResponse{} + err := d.client.call(ctx, "weiyun.download", downloadArgs{ + Items: []downloadFileItem{{FileID: target.FileID, PdirKey: target.ParentKey}}, + }, &resp) + if err != nil { + return nil, err + } + if err = responseError(resp.toolResponse); err != nil { + return nil, err + } + item, err := findDownloadItem(resp.Items, target.FileID) + if err != nil { + return nil, err + } + return &model.Link{ + URL: item.HTTPSDownloadURL, + Header: http.Header{ + "Cookie": []string{item.Cookie}, + }, + }, nil +} + +func (d *WeiYunOpen) Remove(ctx context.Context, obj model.Obj) error { + switch target := obj.(type) { + case *File: + return d.removeFile(ctx, target) + case *Folder: + if target.Root { + return errs.NotSupport + } + return d.removeFolder(ctx, target) + default: + return errs.NotSupport + } +} + +func (d *WeiYunOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + return nil, errors.New("weiyun official MCP does not support directory creation") +} + +func (d *WeiYunOpen) discoverRoot(ctx context.Context) (*Folder, error) { + root := &Folder{Root: true, DirKey: d.RootDirKey, ParentKey: d.RootPDirKey, DirName: defaultRootName} + page, err := d.listPage(ctx, root, 0) + if err != nil { + return nil, err + } + return newRootFolder(page.PdirKey, d.RootPDirKey), nil +} + +func (d *WeiYunOpen) listPage(ctx context.Context, folder *Folder, offset int) (*listResponse, error) { + resp := listResponse{} + err := d.client.call(ctx, "weiyun.list", d.newListArgs(folder, offset), &resp) + if err != nil { + return nil, err + } + if err = responseError(resp.toolResponse); err != nil { + return nil, err + } + if resp.PdirKey == "" { + return nil, errors.New("weiyun list returned empty pdir_key") + } + return &resp, nil +} + +func (d *WeiYunOpen) newListArgs(folder *Folder, offset int) listArgs { + args := listArgs{ + Offset: uint32(offset), + Limit: listPageSize, + OrderBy: d.orderByCode(), + Asc: d.OrderDirection == "asc", + } + if folder.Root && d.RootDirKey == "" { + return args + } + args.DirKey = folder.DirKey + args.PdirKey = folder.ParentKey + return args +} + +func (d *WeiYunOpen) orderByCode() uint32 { + switch d.OrderBy { + case "name": + return orderByName + case "modified": + return orderByModified + default: + return orderByNone + } +} + +func (d *WeiYunOpen) pageObjects(page *listResponse) []model.Obj { + // According to weiyun/SKILL.md, all follow-up operations must use the + // response top-level pdir_key instead of the entry's own pdir_key field. + objects := make([]model.Obj, 0, len(page.DirList)+len(page.FileList)) + for _, item := range page.DirList { + objects = append(objects, newFolder(page.PdirKey, item)) + } + for _, item := range page.FileList { + objects = append(objects, newFile(page.PdirKey, item)) + } + return objects +} + +func (d *WeiYunOpen) removeFile(ctx context.Context, file *File) error { + resp := deleteResponse{} + err := d.client.call(ctx, "weiyun.delete", deleteArgs{ + FileList: []deleteFileItem{{FileID: file.FileID, PdirKey: file.ParentKey}}, + DeleteCompletely: d.DeleteCompletely, + }, &resp) + if err != nil { + return err + } + return responseError(resp.toolResponse) +} + +func (d *WeiYunOpen) removeFolder(ctx context.Context, folder *Folder) error { + resp := deleteResponse{} + err := d.client.call(ctx, "weiyun.delete", deleteArgs{ + DirList: []deleteDirItem{{DirKey: folder.DirKey, PdirKey: folder.ParentKey}}, + DeleteCompletely: d.DeleteCompletely, + }, &resp) + if err != nil { + return err + } + return responseError(resp.toolResponse) +} + +func responseError(resp toolResponse) error { + if resp.Error == "" { + return nil + } + return errors.New(resp.Error) +} + +func findDownloadItem(items []downloadResultItem, fileID string) (*downloadResultItem, error) { + for i := range items { + if items[i].FileID == fileID { + return &items[i], nil + } + } + return nil, fmt.Errorf("weiyun download result missing file %s", fileID) +} + +var _ driver.Driver = (*WeiYunOpen)(nil) +var _ driver.GetRooter = (*WeiYunOpen)(nil) +var _ driver.MkdirResult = (*WeiYunOpen)(nil) +var _ driver.Remove = (*WeiYunOpen)(nil) diff --git a/drivers/weiyun_open/hash.go b/drivers/weiyun_open/hash.go new file mode 100644 index 000000000..b15f6988a --- /dev/null +++ b/drivers/weiyun_open/hash.go @@ -0,0 +1,248 @@ +package weiyun_open + +import ( + "crypto/md5" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "fmt" + "io" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +type sha1State struct { + h [5]uint32 + buffer []byte + messageBytes uint64 +} + +func newSHA1State() *sha1State { + return &sha1State{ + h: [5]uint32{ + 0x67452301, + 0xEFCDAB89, + 0x98BADCFE, + 0x10325476, + 0xC3D2E1F0, + }, + buffer: make([]byte, 0, 64), + } +} + +func (s *sha1State) Update(data []byte) { + s.messageBytes += uint64(len(data)) + s.buffer = append(s.buffer, data...) + for len(s.buffer) >= 64 { + s.processChunk(s.buffer[:64]) + s.buffer = s.buffer[64:] + } +} + +func (s *sha1State) processChunk(chunk []byte) { + var words [80]uint32 + for i := 0; i < 16; i++ { + words[i] = binary.BigEndian.Uint32(chunk[i*4 : (i+1)*4]) + } + for i := 16; i < len(words); i++ { + words[i] = rol(words[i-3]^words[i-8]^words[i-14]^words[i-16], 1) + } + a, b, c, d, e := s.h[0], s.h[1], s.h[2], s.h[3], s.h[4] + for i := 0; i < len(words); i++ { + f, k := sha1Round(b, c, d, i) + temp := rol(a, 5) + f + e + k + words[i] + e, d, c, b, a = d, c, rol(b, 30), a, temp + } + s.h[0] += a + s.h[1] += b + s.h[2] += c + s.h[3] += d + s.h[4] += e +} + +func (s *sha1State) StateHex() (string, error) { + if len(s.buffer) != 0 { + return "", fmt.Errorf("sha1 state is not block-aligned: %d", len(s.buffer)) + } + out := make([]byte, 0, 20) + for _, value := range s.h { + var buf [4]byte + binary.LittleEndian.PutUint32(buf[:], value) + out = append(out, buf[:]...) + } + return hex.EncodeToString(out), nil +} + +func (s *sha1State) SumHex() string { + clone := *s + clone.buffer = append([]byte(nil), s.buffer...) + padding := sha1Padding(clone.messageBytes, len(clone.buffer)) + clone.Update(padding) + out := make([]byte, 20) + for i, value := range clone.h { + binary.BigEndian.PutUint32(out[i*4:(i+1)*4], value) + } + return hex.EncodeToString(out) +} + +func buildUploadRequest(file model.File, fileName string, fileSize int64, pdirKey string) (*preUploadArgs, error) { + if fileSize < 0 { + return nil, fmt.Errorf("weiyun open upload does not support unknown file size") + } + if fileSize == 0 { + return emptyUploadRequest(fileName, pdirKey), nil + } + lastBlockSize := blockTailSize(fileSize) + checkBlockSize := blockCheckSize(lastBlockSize) + beforeBlockSize := fileSize - lastBlockSize + md5Hash := md5.New() + sha1Hash := newSHA1State() + blockSHAList, err := collectBlockHashes(file, beforeBlockSize, sha1Hash, md5Hash) + if err != nil { + return nil, err + } + checkSHA, checkData, fileSHA, err := finishLastBlock( + file, beforeBlockSize, lastBlockSize, checkBlockSize, sha1Hash, md5Hash, + ) + if err != nil { + return nil, err + } + blockSHAList = append(blockSHAList, fileSHA) + return &preUploadArgs{ + FileName: fileName, + FileSize: uint64(fileSize), + FileSHA: fileSHA, + FileMD5: hex.EncodeToString(md5Hash.Sum(nil)), + BlockSHAList: blockSHAList, + CheckSHA: checkSHA, + CheckData: checkData, + PdirKey: pdirKey, + }, nil +} + +func emptyUploadRequest(fileName string, pdirKey string) *preUploadArgs { + return &preUploadArgs{ + FileName: fileName, + FileSize: 0, + FileSHA: emptyFileSHA1, + FileMD5: emptyFileMD5, + BlockSHAList: []string{emptyFileSHA1}, + CheckSHA: emptySHA1StateHex, + CheckData: "", + PdirKey: pdirKey, + } +} + +func collectBlockHashes( + file model.File, + beforeBlockSize int64, + sha1Hash *sha1State, + md5Hash io.Writer, +) ([]string, error) { + blockCount := int(beforeBlockSize/uploadBlockSize) + 1 + blockSHAList := make([]string, 0, blockCount) + for offset := int64(0); offset < beforeBlockSize; offset += uploadBlockSize { + chunk, err := readChunk(file, offset, uploadBlockSize) + if err != nil { + return nil, err + } + sha1Hash.Update(chunk) + if _, err = md5Hash.Write(chunk); err != nil { + return nil, err + } + state, err := sha1Hash.StateHex() + if err != nil { + return nil, err + } + blockSHAList = append(blockSHAList, state) + } + return blockSHAList, nil +} + +func finishLastBlock( + file model.File, + beforeBlockSize int64, + lastBlockSize int64, + checkBlockSize int64, + sha1Hash *sha1State, + md5Hash io.Writer, +) (string, string, string, error) { + middleSize := lastBlockSize - checkBlockSize + if middleSize > 0 { + middle, err := readChunk(file, beforeBlockSize, middleSize) + if err != nil { + return "", "", "", err + } + sha1Hash.Update(middle) + if _, err = md5Hash.Write(middle); err != nil { + return "", "", "", err + } + } + checkSHA, err := sha1Hash.StateHex() + if err != nil { + return "", "", "", err + } + tail, err := readChunk(file, beforeBlockSize+middleSize, checkBlockSize) + if err != nil { + return "", "", "", err + } + sha1Hash.Update(tail) + if _, err = md5Hash.Write(tail); err != nil { + return "", "", "", err + } + return checkSHA, base64.StdEncoding.EncodeToString(tail), sha1Hash.SumHex(), nil +} + +func readChunk(file model.File, offset int64, length int64) ([]byte, error) { + reader := io.NewSectionReader(file, offset, length) + data, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + if int64(len(data)) != length { + return nil, fmt.Errorf("expected %d bytes, got %d", length, len(data)) + } + return data, nil +} + +func blockTailSize(fileSize int64) int64 { + tail := fileSize % uploadBlockSize + if tail == 0 { + return uploadBlockSize + } + return tail +} + +func blockCheckSize(lastBlockSize int64) int64 { + size := lastBlockSize % checkBlockDivisor + if size == 0 { + return checkBlockDivisor + } + return size +} + +func sha1Round(b, c, d uint32, index int) (uint32, uint32) { + switch { + case index < 20: + return (b & c) | (^b & d), 0x5A827999 + case index < 40: + return b ^ c ^ d, 0x6ED9EBA1 + case index < 60: + return (b & c) | (b & d) | (c & d), 0x8F1BBCDC + default: + return b ^ c ^ d, 0xCA62C1D6 + } +} + +func sha1Padding(messageBytes uint64, buffered int) []byte { + padding := []byte{0x80} + zeros := (56 - (buffered+1)%64 + 64) % 64 + padding = append(padding, make([]byte, zeros)...) + var length [8]byte + binary.BigEndian.PutUint64(length[:], messageBytes*8) + return append(padding, length[:]...) +} + +func rol(value uint32, bits uint) uint32 { + return value<>(32-bits) +} diff --git a/drivers/weiyun_open/hash_test.go b/drivers/weiyun_open/hash_test.go new file mode 100644 index 000000000..56f054f2b --- /dev/null +++ b/drivers/weiyun_open/hash_test.go @@ -0,0 +1,25 @@ +package weiyun_open + +import "testing" + +func TestBuildUploadRequestForEmptyFile(t *testing.T) { + req, err := buildUploadRequest(nil, "empty.txt", 0, "parent-key") + if err != nil { + t.Fatalf("build upload request: %v", err) + } + if req.FileSHA != emptyFileSHA1 { + t.Fatalf("unexpected file sha: %s", req.FileSHA) + } + if req.FileMD5 != emptyFileMD5 { + t.Fatalf("unexpected file md5: %s", req.FileMD5) + } + if req.CheckSHA != emptySHA1StateHex { + t.Fatalf("unexpected check sha: %s", req.CheckSHA) + } + if len(req.BlockSHAList) != 1 || req.BlockSHAList[0] != emptyFileSHA1 { + t.Fatalf("unexpected block sha list: %#v", req.BlockSHAList) + } + if req.PdirKey != "parent-key" { + t.Fatalf("unexpected parent key: %s", req.PdirKey) + } +} diff --git a/drivers/weiyun_open/json_number.go b/drivers/weiyun_open/json_number.go new file mode 100644 index 000000000..4a3102d31 --- /dev/null +++ b/drivers/weiyun_open/json_number.go @@ -0,0 +1,72 @@ +package weiyun_open + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" +) + +type jsonInt64 int64 +type jsonUint32 uint32 +type jsonUint64 uint64 + +func (n *jsonInt64) UnmarshalJSON(data []byte) error { + value, err := parseJSONInteger(data) + if err != nil { + return err + } + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return fmt.Errorf("parse int64 %q: %w", value, err) + } + *n = jsonInt64(parsed) + return nil +} + +func (n *jsonUint32) UnmarshalJSON(data []byte) error { + value, err := parseJSONInteger(data) + if err != nil { + return err + } + parsed, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return fmt.Errorf("parse uint32 %q: %w", value, err) + } + *n = jsonUint32(parsed) + return nil +} + +func (n *jsonUint64) UnmarshalJSON(data []byte) error { + value, err := parseJSONInteger(data) + if err != nil { + return err + } + parsed, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return fmt.Errorf("parse uint64 %q: %w", value, err) + } + *n = jsonUint64(parsed) + return nil +} + +func parseJSONInteger(data []byte) (string, error) { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 { + return "", fmt.Errorf("unexpected empty integer") + } + if bytes.Equal(trimmed, []byte("null")) { + return "", fmt.Errorf("unexpected null integer") + } + if trimmed[0] != '"' { + return string(trimmed), nil + } + value := "" + if err := json.Unmarshal(trimmed, &value); err != nil { + return "", err + } + if value == "" { + return "", fmt.Errorf("unexpected empty integer string") + } + return value, nil +} diff --git a/drivers/weiyun_open/json_number_test.go b/drivers/weiyun_open/json_number_test.go new file mode 100644 index 000000000..b499bc024 --- /dev/null +++ b/drivers/weiyun_open/json_number_test.go @@ -0,0 +1,62 @@ +package weiyun_open + +import ( + "encoding/json" + "testing" +) + +func TestListResponseUnmarshalStringNumbers(t *testing.T) { + data := []byte(`{ + "pdir_key": "parent", + "total_dir_count": "1", + "total_file_count": 1, + "dir_list": [{ + "dir_key": "dir-1", + "dir_name": "docs", + "dir_ctime": "1712312312000", + "dir_mtime": 1712312313000 + }], + "file_list": [{ + "file_id": "file-1", + "filename": "readme.txt", + "file_size": "12", + "file_ctime": "1712312314000", + "file_mtime": 1712312315000, + "pdir_key": "ignored" + }], + "finish_flag": true + }`) + + resp := listResponse{} + if err := json.Unmarshal(data, &resp); err != nil { + t.Fatalf("unmarshal list response: %v", err) + } + if got := int64(resp.DirList[0].DirCTime); got != 1712312312000 { + t.Fatalf("unexpected dir_ctime: %d", got) + } + if got := int64(resp.FileList[0].FileSize); got != 12 { + t.Fatalf("unexpected file_size: %d", got) + } + if got := uint32(resp.TotalDirCount); got != 1 { + t.Fatalf("unexpected total_dir_count: %d", got) + } +} + +func TestDeleteResponseUnmarshalStringNumbers(t *testing.T) { + data := []byte(`{ + "freed_space": "13", + "freed_index_cnt": "1", + "error": "" + }`) + + resp := deleteResponse{} + if err := json.Unmarshal(data, &resp); err != nil { + t.Fatalf("unmarshal delete response: %v", err) + } + if got := int64(resp.FreedSpace); got != 13 { + t.Fatalf("unexpected freed_space: %d", got) + } + if got := uint32(resp.FreedIndexCnt); got != 1 { + t.Fatalf("unexpected freed_index_cnt: %d", got) + } +} diff --git a/drivers/weiyun_open/meta.go b/drivers/weiyun_open/meta.go new file mode 100644 index 000000000..3ce5e32f0 --- /dev/null +++ b/drivers/weiyun_open/meta.go @@ -0,0 +1,29 @@ +package weiyun_open + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + MCPToken string `json:"mcp_token" required:"true"` + EnvID string `json:"env_id"` + APIURL string `json:"api_url" default:"https://www.weiyun.com/api/v3/mcpserver"` + RootDirKey string `json:"root_dir_key"` + RootPDirKey string `json:"root_pdir_key"` + OrderBy string `json:"order_by" type:"select" options:"name,modified,none" default:"name"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + DeleteCompletely bool `json:"delete_completely" type:"bool" default:"false"` +} + +var config = driver.Config{ + Name: "WeiYun Open", + OnlyProxy: true, + CheckStatus: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &WeiYunOpen{} + }) +} diff --git a/drivers/weiyun_open/types.go b/drivers/weiyun_open/types.go new file mode 100644 index 000000000..004fcd541 --- /dev/null +++ b/drivers/weiyun_open/types.go @@ -0,0 +1,233 @@ +package weiyun_open + +import ( + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +const ( + defaultAPIURL = "https://www.weiyun.com/api/v3/mcpserver" + defaultRootName = "/" + listPageSize = 50 + maxUploadRounds = 200 + uploadBlockSize = 512 * 1024 + checkBlockDivisor = 128 + cacheProgressEnd = 10 + uploadStateDone = 2 + emptyFileSHA1 = "da39a3ee5e6b4b0d3255bfef95601890afd80709" + emptyFileMD5 = "d41d8cd98f00b204e9800998ecf8427e" + emptySHA1StateHex = "0123456789abcdeffedcba9876543210f0e1d2c3" +) + +const ( + orderByNone = iota + orderByName + orderByModified +) + +type toolResponse struct { + Error string `json:"error"` +} + +type listArgs struct { + GetType uint32 `json:"get_type,omitempty"` + Offset uint32 `json:"offset,omitempty"` + Limit uint32 `json:"limit"` + OrderBy uint32 `json:"order_by,omitempty"` + Asc bool `json:"asc,omitempty"` + DirKey string `json:"dir_key,omitempty"` + PdirKey string `json:"pdir_key,omitempty"` +} + +type dirItem struct { + DirKey string `json:"dir_key"` + DirName string `json:"dir_name"` + DirCTime jsonInt64 `json:"dir_ctime"` + DirMTime jsonInt64 `json:"dir_mtime"` +} + +type fileItem struct { + FileID string `json:"file_id"` + FileName string `json:"filename"` + FileSize jsonInt64 `json:"file_size"` + FileCTime jsonInt64 `json:"file_ctime"` + FileMTime jsonInt64 `json:"file_mtime"` + PdirKey string `json:"pdir_key"` +} + +type listResponse struct { + toolResponse + PdirKey string `json:"pdir_key"` + TotalDirCount jsonUint32 `json:"total_dir_count"` + TotalFileCount jsonUint32 `json:"total_file_count"` + DirList []dirItem `json:"dir_list"` + FileList []fileItem `json:"file_list"` + FinishFlag bool `json:"finish_flag"` +} + +type downloadFileItem struct { + FileID string `json:"file_id"` + PdirKey string `json:"pdir_key"` +} + +type downloadArgs struct { + Items []downloadFileItem `json:"items"` +} + +type downloadResultItem struct { + FileID string `json:"file_id"` + HTTPSDownloadURL string `json:"https_download_url"` + FileSize jsonInt64 `json:"file_size"` + Cookie string `json:"cookie"` +} + +type downloadResponse struct { + toolResponse + Items []downloadResultItem `json:"items"` +} + +type deleteFileItem struct { + FileID string `json:"file_id"` + PdirKey string `json:"pdir_key"` +} + +type deleteDirItem struct { + DirKey string `json:"dir_key"` + PdirKey string `json:"pdir_key"` +} + +type deleteArgs struct { + FileList []deleteFileItem `json:"file_list,omitempty"` + DirList []deleteDirItem `json:"dir_list,omitempty"` + DeleteCompletely bool `json:"delete_completely"` +} + +type deleteResponse struct { + toolResponse + FreedSpace jsonInt64 `json:"freed_space"` + FreedIndexCnt jsonUint32 `json:"freed_index_cnt"` +} + +type uploadChannel struct { + ID jsonUint32 `json:"id"` + Offset jsonUint64 `json:"offset"` + Len jsonUint32 `json:"len"` +} + +type preUploadArgs struct { + FileName string `json:"filename"` + FileSize uint64 `json:"file_size"` + FileSHA string `json:"file_sha"` + FileMD5 string `json:"file_md5,omitempty"` + BlockSHAList []string `json:"block_sha_list"` + CheckSHA string `json:"check_sha"` + CheckData string `json:"check_data,omitempty"` + PdirKey string `json:"pdir_key,omitempty"` +} + +type uploadChunkArgs struct { + FileName string `json:"filename"` + FileSize uint64 `json:"file_size"` + FileSHA string `json:"file_sha"` + BlockSHAList []string `json:"block_sha_list"` + CheckSHA string `json:"check_sha"` + UploadKey string `json:"upload_key"` + ChannelList []uploadChannel `json:"channel_list"` + ChannelID uint32 `json:"channel_id"` + Ex string `json:"ex"` + FileData []byte `json:"file_data"` +} + +type uploadResponse struct { + toolResponse + FileID string `json:"file_id"` + FileName string `json:"filename"` + FileExist bool `json:"file_exist"` + UploadState jsonInt64 `json:"upload_state"` + UploadKey string `json:"upload_key"` + ChannelList []uploadChannel `json:"channel_list"` + Ex string `json:"ex"` +} + +type File struct { + ParentKey string + FileID string + FileName string + FileSize int64 + FileCTime int64 + FileMTime int64 +} + +func newFile(parentKey string, item fileItem) *File { + return &File{ + ParentKey: parentKey, + FileID: item.FileID, + FileName: item.FileName, + FileSize: int64(item.FileSize), + FileCTime: int64(item.FileCTime), + FileMTime: int64(item.FileMTime), + } +} + +func (f *File) CreateTime() time.Time { return time.UnixMilli(f.FileCTime) } +func (f *File) GetHash() utils.HashInfo { return utils.HashInfo{} } +func (f *File) GetID() string { return f.FileID } +func (f *File) GetName() string { return f.FileName } +func (f *File) GetPath() string { return "" } +func (f *File) GetSize() int64 { return f.FileSize } +func (f *File) IsDir() bool { return false } +func (f *File) ModTime() time.Time { return time.UnixMilli(f.FileMTime) } + +type Folder struct { + Root bool + ParentKey string + DirKey string + DirName string + DirCTime int64 + DirMTime int64 +} + +func newFolder(parentKey string, item dirItem) *Folder { + return &Folder{ + ParentKey: parentKey, + DirKey: item.DirKey, + DirName: item.DirName, + DirCTime: int64(item.DirCTime), + DirMTime: int64(item.DirMTime), + } +} + +func newRootFolder(currentKey, parentKey string) *Folder { + return &Folder{ + Root: true, + ParentKey: parentKey, + DirKey: currentKey, + DirName: defaultRootName, + } +} + +func (f *Folder) CreateTime() time.Time { return time.UnixMilli(f.DirCTime) } +func (f *Folder) GetHash() utils.HashInfo { return utils.HashInfo{} } +func (f *Folder) GetID() string { return f.DirKey } +func (f *Folder) GetName() string { return f.DirName } +func (f *Folder) GetPath() string { return "" } +func (f *Folder) GetSize() int64 { return 0 } +func (f *Folder) IsDir() bool { return true } +func (f *Folder) ModTime() time.Time { return time.UnixMilli(f.DirMTime) } + +func fileFromUpload(parentKey string, resp *uploadResponse, size int64) *File { + now := time.Now().UnixMilli() + return &File{ + ParentKey: parentKey, + FileID: resp.FileID, + FileName: resp.FileName, + FileSize: size, + FileCTime: now, + FileMTime: now, + } +} + +var _ model.Obj = (*File)(nil) +var _ model.Obj = (*Folder)(nil) diff --git a/drivers/weiyun_open/upload.go b/drivers/weiyun_open/upload.go new file mode 100644 index 000000000..fa1a5d247 --- /dev/null +++ b/drivers/weiyun_open/upload.go @@ -0,0 +1,140 @@ +package weiyun_open + +import ( + "context" + "fmt" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func (d *WeiYunOpen) Put( + ctx context.Context, + dstDir model.Obj, + stream model.FileStreamer, + up driver.UpdateProgress, +) (model.Obj, error) { + folder, ok := dstDir.(*Folder) + if !ok { + return nil, errs.NotSupport + } + cacheUp := model.UpdateProgressWithRange(up, 0, cacheProgressEnd) + file, err := stream.CacheFullAndWriter(&cacheUp, nil) + if err != nil { + return nil, err + } + req, err := buildUploadRequest(file, stream.GetName(), stream.GetSize(), folder.DirKey) + if err != nil { + return nil, err + } + uploadUp := model.UpdateProgressWithRange(up, cacheProgressEnd, 100) + resp, err := d.uploadFile(ctx, file, req, uploadUp) + if err != nil { + return nil, err + } + return d.finalizeUpload(ctx, folder, resp, stream.GetName(), stream.GetSize()) +} + +func (d *WeiYunOpen) uploadFile( + ctx context.Context, + file model.File, + req *preUploadArgs, + up driver.UpdateProgress, +) (*uploadResponse, error) { + for round := 0; round < maxUploadRounds; round++ { + resp, err := d.preUpload(ctx, req) + if err != nil { + return nil, err + } + if resp.FileExist || resp.UploadState == uploadStateDone { + up(100) + return resp, nil + } + channel, ok := pendingChannel(resp.ChannelList) + if !ok { + return nil, fmt.Errorf("weiyun upload returned no pending channel, state=%d", resp.UploadState) + } + chunk, err := readChunk(file, int64(channel.Offset), int64(channel.Len)) + if err != nil { + return nil, err + } + resp, err = d.uploadChunk(ctx, req, resp, channel, chunk) + if err != nil { + return nil, err + } + if resp.UploadState == uploadStateDone { + up(100) + return resp, nil + } + up(uploadProgress(channel, len(chunk), req.FileSize)) + } + return nil, fmt.Errorf("weiyun upload exceeded %d rounds", maxUploadRounds) +} + +func (d *WeiYunOpen) preUpload(ctx context.Context, req *preUploadArgs) (*uploadResponse, error) { + resp := uploadResponse{} + err := d.client.call(ctx, "weiyun.upload", req, &resp) + if err != nil { + return nil, err + } + if err = responseError(resp.toolResponse); err != nil { + return nil, err + } + if resp.FileName == "" { + resp.FileName = req.FileName + } + return &resp, nil +} + +func (d *WeiYunOpen) uploadChunk( + ctx context.Context, + req *preUploadArgs, + state *uploadResponse, + channel uploadChannel, + chunk []byte, +) (*uploadResponse, error) { + resp := uploadResponse{} + args := uploadChunkArgs{ + FileName: req.FileName, + FileSize: req.FileSize, + FileSHA: req.FileSHA, + BlockSHAList: []string{}, + CheckSHA: req.CheckSHA, + UploadKey: state.UploadKey, + ChannelList: state.ChannelList, + ChannelID: uint32(channel.ID), + Ex: state.Ex, + FileData: chunk, + } + err := d.client.call(ctx, "weiyun.upload", args, &resp) + if err != nil { + return nil, err + } + if err = responseError(resp.toolResponse); err != nil { + return nil, err + } + if resp.FileName == "" { + resp.FileName = req.FileName + } + return &resp, nil +} + +func pendingChannel(channels []uploadChannel) (uploadChannel, bool) { + for _, channel := range channels { + if channel.Len > 0 { + return channel, true + } + } + return uploadChannel{}, false +} + +func uploadProgress(channel uploadChannel, chunkLen int, total uint64) float64 { + done := uint64(channel.Offset) + uint64(chunkLen) + if done > total { + done = total + } + return float64(done) * 100 / float64(total) +} + +var _ driver.PutResult = (*WeiYunOpen)(nil) diff --git a/drivers/weiyun_open/upload_result.go b/drivers/weiyun_open/upload_result.go new file mode 100644 index 000000000..62265d8dd --- /dev/null +++ b/drivers/weiyun_open/upload_result.go @@ -0,0 +1,120 @@ +package weiyun_open + +import ( + "context" + "fmt" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func (d *WeiYunOpen) finalizeUpload( + ctx context.Context, + folder *Folder, + resp *uploadResponse, + fallbackName string, + size int64, +) (model.Obj, error) { + if resp.FileID != "" { + return uploadResultFile(folder.DirKey, resp, fallbackName, size), nil + } + file, err := d.findUploadedFile(ctx, folder, uploadCandidateNames(resp.FileName, fallbackName), size) + if err == nil { + return file, nil + } + return nil, fmt.Errorf( + "weiyun upload finished without file_id: state=%d, file_exist=%t, filename=%q: %w", + resp.UploadState, resp.FileExist, uploadFileName(resp.FileName, fallbackName), err, + ) +} + +func (d *WeiYunOpen) findUploadedFile( + ctx context.Context, + folder *Folder, + names []string, + size int64, +) (*File, error) { + offset := 0 + candidates := make([]*File, 0) + for { + page, err := d.listPage(ctx, folder, offset) + if err != nil { + return nil, err + } + candidates = append(candidates, matchedUploadFiles(page, names)...) + if page.FinishFlag { + return pickUploadedFile(candidates, size) + } + pageCount := len(page.DirList) + len(page.FileList) + if pageCount == 0 { + return nil, fmt.Errorf("weiyun list returned empty page before finish") + } + offset += pageCount + } +} + +func matchedUploadFiles(page *listResponse, names []string) []*File { + files := make([]*File, 0) + for _, item := range page.FileList { + if !containsName(names, item.FileName) { + continue + } + files = append(files, newFile(page.PdirKey, item)) + } + return files +} + +func pickUploadedFile(files []*File, size int64) (*File, error) { + if len(files) == 0 { + return nil, fmt.Errorf("uploaded file not found in target directory") + } + best := files[0] + bestScore := uploadCandidateScore(best, size) + for i := 1; i < len(files); i++ { + score := uploadCandidateScore(files[i], size) + if score > bestScore || (score == bestScore && files[i].FileMTime > best.FileMTime) { + best = files[i] + bestScore = score + } + } + return best, nil +} + +func uploadCandidateScore(file *File, size int64) int { + score := 0 + if file.FileSize == size { + score += 2 + } + if file.FileMTime > 0 { + score++ + } + return score +} + +func uploadCandidateNames(primary string, fallback string) []string { + if primary == "" || primary == fallback { + return []string{fallback} + } + return []string{primary, fallback} +} + +func uploadFileName(primary string, fallback string) string { + if primary != "" { + return primary + } + return fallback +} + +func containsName(names []string, name string) bool { + for _, candidate := range names { + if candidate == name { + return true + } + } + return false +} + +func uploadResultFile(parentKey string, resp *uploadResponse, fallbackName string, size int64) *File { + file := fileFromUpload(parentKey, resp, size) + file.FileName = uploadFileName(resp.FileName, fallbackName) + return file +} diff --git a/drivers/weiyun_open/upload_result_test.go b/drivers/weiyun_open/upload_result_test.go new file mode 100644 index 000000000..a3f488cba --- /dev/null +++ b/drivers/weiyun_open/upload_result_test.go @@ -0,0 +1,26 @@ +package weiyun_open + +import "testing" + +func TestPickUploadedFilePrefersMatchingSizeAndNewerFile(t *testing.T) { + files := []*File{ + {FileID: "older-match", FileName: "demo.txt", FileSize: 12, FileMTime: 10}, + {FileID: "size-miss", FileName: "demo.txt", FileSize: 99, FileMTime: 99}, + {FileID: "newer-match", FileName: "demo.txt", FileSize: 12, FileMTime: 20}, + } + + got, err := pickUploadedFile(files, 12) + if err != nil { + t.Fatalf("pick uploaded file: %v", err) + } + if got.FileID != "newer-match" { + t.Fatalf("unexpected file picked: %s", got.FileID) + } +} + +func TestUploadCandidateNamesDeduplicatesFallback(t *testing.T) { + names := uploadCandidateNames("demo.txt", "demo.txt") + if len(names) != 1 || names[0] != "demo.txt" { + t.Fatalf("unexpected names: %#v", names) + } +}