Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 17 additions & 31 deletions .github/workflows/test_docker.yml
Original file line number Diff line number Diff line change
@@ -1,57 +1,56 @@
name: Beta Release (Docker)

on:
workflow_dispatch:
push:
branches:
- main
pull_request:
branches:
- main
- fix # 👈 允许你的 fix 分支触发

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

env:
DOCKERHUB_ORG_NAME: ${{ vars.DOCKERHUB_ORG_NAME || 'openlistteam' }}
GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'openlistteam' }}
IMAGE_NAME: openlist-git
IMAGE_NAME_DOCKERHUB: openlist
GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'ironboxplus' }} # 👈 最好改成你的用户名,防止推错地方
IMAGE_NAME: openlist
REGISTRY: ghcr.io
ARTIFACT_NAME: 'binaries_docker_release'
RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/ppc64le,linux/riscv64,linux/loong64' ### Temporarily disable Docker builds for linux/s390x architectures for unknown reasons.
IMAGE_PUSH: ${{ github.event_name == 'push' }}
# 👇 关键修改:只保留 linux/amd64,删掉后面一长串
RELEASE_PLATFORMS: 'linux/amd64'
# 👇 关键修改:强制允许推送,不用管是不是 push 事件
IMAGE_PUSH: 'true'
IMAGE_TAGS_BETA: |
type=ref,event=pr
type=raw,value=beta,enable={{is_default_branch}}
type=raw,value=beta

jobs:
build_binary:
name: Build Binaries for Docker Release (Beta)
name: Build Binaries (x64 Only)
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4

- uses: actions/setup-go@v5
with:
go-version: '1.25.0'

# 即使只构建 x64,我们也需要 musl 工具链(因为 BuildDockerMultiplatform 默认会检查它)
- name: Cache Musl
id: cache-musl
uses: actions/cache@v4
with:
path: build/musl-libs
key: docker-musl-libs-v2

- name: Download Musl Library
if: steps.cache-musl.outputs.cache-hit != 'true'
run: bash build.sh prepare docker-multiplatform
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- name: Build go binary (beta)
- name: Build go binary
# 这里还是跑 docker-multiplatform,虽然会多编译一些架构,但这是兼容 Dockerfile 路径最稳妥的方法
run: bash build.sh beta docker-multiplatform
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Expand All @@ -69,12 +68,13 @@ jobs:

release_docker:
needs: build_binary
name: Release Docker image (Beta)
name: Release Docker (x64)
runs-on: ubuntu-latest
permissions:
packages: write
strategy:
matrix:
# 你可以选择只构建 latest,或者保留全部变体
image: ["latest", "ffmpeg", "aria2", "aio"]
include:
- image: "latest"
Expand Down Expand Up @@ -102,46 +102,32 @@ jobs:
with:
name: ${{ env.ARTIFACT_NAME }}
path: 'build/'

- name: Set up QEMU
uses: docker/setup-qemu-action@v3

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

# 👇 只保留 GitHub 登录,删除了 DockerHub 登录
- name: Login to GitHub Container Registry
if: env.IMAGE_PUSH == 'true'
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Login to DockerHub Container Registry
if: env.IMAGE_PUSH == 'true'
uses: docker/login-action@v3
with:
username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}

- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: |
${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }}
${{ env.DOCKERHUB_ORG_NAME }}/${{ env.IMAGE_NAME_DOCKERHUB }}
tags: ${{ env.IMAGE_TAGS_BETA }}
flavor: |
${{ matrix.tag_favor }}
flavor: ${{ matrix.tag_favor }}

- name: Build and push
id: docker_build
uses: docker/build-push-action@v6
with:
context: .
file: Dockerfile.ci
push: ${{ env.IMAGE_PUSH == 'true' }}
push: true
build-args: |
BASE_IMAGE_TAG=${{ matrix.base_image_tag }}
${{ matrix.build_arg }}
Expand Down
20 changes: 10 additions & 10 deletions build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -186,8 +186,8 @@ BuildDockerMultiplatform() {
docker_lflags="--extldflags '-static -fpic' $ldflags"
export CGO_ENABLED=1

OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-riscv64 linux-ppc64le linux-loong64) ## Disable linux-s390x builds
CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc riscv64-linux-musl-gcc powerpc64le-linux-musl-gcc loongarch64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds
OS_ARCHES=(linux-amd64) ## Disable linux-s390x builds
CGO_ARGS=(x86_64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds
for i in "${!OS_ARCHES[@]}"; do
os_arch=${OS_ARCHES[$i]}
cgo_cc=${CGO_ARGS[$i]}
Expand All @@ -205,14 +205,14 @@ BuildDockerMultiplatform() {
GO_ARM=(6 7)
export GOOS=linux
export GOARCH=arm
for i in "${!DOCKER_ARM_ARCHES[@]}"; do
docker_arch=${DOCKER_ARM_ARCHES[$i]}
cgo_cc=${CGO_ARGS[$i]}
export GOARM=${GO_ARM[$i]}
export CC=${cgo_cc}
echo "building for $docker_arch"
go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter .
done
# for i in "${!DOCKER_ARM_ARCHES[@]}"; do
# docker_arch=${DOCKER_ARM_ARCHES[$i]}
# cgo_cc=${CGO_ARGS[$i]}
# export GOARM=${GO_ARM[$i]}
# export CC=${cgo_cc}
# echo "building for $docker_arch"
# go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter .
# done
}

BuildRelease() {
Expand Down
3 changes: 2 additions & 1 deletion drivers/115_open/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,8 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre
}
sha1 := file.GetHash().GetHash(utils.SHA1)
if len(sha1) != utils.SHA1.Width {
_, sha1, err = stream.CacheFullAndHash(file, &up, utils.SHA1)
// 流式计算SHA1
sha1, err = stream.StreamHashFile(file, utils.SHA1, 10, &up)
if err != nil {
return err
}
Expand Down
39 changes: 33 additions & 6 deletions drivers/123_open/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,20 +156,44 @@ func (d *Open123) Remove(ctx context.Context, obj model.Obj) error {
}

func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
// 1. 创建文件
// 1. 准备参数
// parentFileID 父目录id,上传到根目录时填写 0
parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64)
if err != nil {
return nil, fmt.Errorf("parse parentFileID error: %v", err)
}

// etag 文件md5
etag := file.GetHash().GetHash(utils.MD5)
if len(etag) < utils.MD5.Width {
_, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5)
if len(etag) >= utils.MD5.Width {
// 有etag时,先尝试秒传
createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false)
if err != nil {
return nil, err
}
// 是否秒传
if createResp.Data.Reuse {
// 秒传成功才会返回正确的 FileID,否则为 0
if createResp.Data.FileID != 0 {
return File{
FileName: file.GetName(),
Size: file.GetSize(),
FileId: createResp.Data.FileID,
Type: 2,
Etag: etag,
}, nil
}
}
// 秒传失败,etag可能不可靠,继续流式计算真实MD5
}

// 流式MD5计算
etag, err = stream.StreamHashFile(file, utils.MD5, 40, &up)
if err != nil {
return nil, err
}

// 2. 创建上传任务
createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false)
if err != nil {
return nil, err
Expand All @@ -188,13 +212,16 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre
}
}

// 2. 上传分片
err = d.Upload(ctx, file, createResp, up)
// 3. 上传分片
uploadProgress := func(p float64) {
up(40 + p*0.6)
}
err = d.Upload(ctx, file, createResp, uploadProgress)
if err != nil {
return nil, err
}

// 3. 上传完毕
// 4. 合并分片/完成上传
for range 60 {
uploadCompleteResp, err := d.complete(createResp.Data.PreuploadID)
// 返回错误代码未知,如:20103,文档也没有具体说
Expand Down
5 changes: 1 addition & 4 deletions drivers/baidu_netdisk/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"time"

"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)

var (
Expand Down Expand Up @@ -76,9 +75,7 @@ func fileToObj(f File) *model.ObjThumb {
Modified: time.Unix(f.ServerMtime, 0),
Ctime: time.Unix(f.ServerCtime, 0),
IsFolder: f.Isdir == 1,

// 直接获取的MD5是错误的
HashInfo: utils.NewHashInfo(utils.MD5, DecryptMd5(f.Md5)),
// 百度API返回的MD5不可信,不使用HashInfo
},
Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3},
}
Expand Down
34 changes: 34 additions & 0 deletions internal/stream/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,40 @@ func CacheFullAndHash(stream model.FileStreamer, up *model.UpdateProgress, hashT
return tmpF, hex.EncodeToString(h.Sum(nil)), nil
}

// StreamHashFile 流式计算文件哈希值,避免将整个文件加载到内存
// file: 文件流
// hashType: 哈希算法类型
// progressWeight: 进度权重(0-100),用于计算整体进度
// up: 进度回调函数
func StreamHashFile(file model.FileStreamer, hashType *utils.HashType, progressWeight float64, up *model.UpdateProgress) (string, error) {
hashFunc := hashType.NewFunc()
size := file.GetSize()
chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk
var offset int64 = 0

for offset < size {
readSize := chunkSize
if size-offset < chunkSize {
readSize = size - offset
}
reader, err := file.RangeRead(http_range.Range{Start: offset, Length: readSize})
if err != nil {
return "", fmt.Errorf("range read for hash calculation failed: %w", err)
}
if _, err := io.Copy(hashFunc, reader); err != nil {
return "", fmt.Errorf("calculate hash failed: %w", err)
}
offset += readSize

if up != nil && progressWeight > 0 {
progress := progressWeight * float64(offset) / float64(size)
(*up)(progress)
}
}

return hex.EncodeToString(hashFunc.Sum(nil)), nil
}

type StreamSectionReaderIF interface {
// 线程不安全
GetSectionReader(off, length int64) (io.ReadSeeker, error)
Expand Down