Skip to content

feat: separate MinIO write path from public URL #999

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
rondinelisaad wants to merge 1 commit into from
Closed

feat: separate MinIO write path from public URL #999

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions files_storage/migrations/0005_minioconfiguration_public_read_write_prefix_and_more.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Generated by Codex on 2026-06-19

from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("files_storage", "0004_remove_minioconfiguration_bucket_app_subdir_and_more"),
]

operations = [
migrations.AddField(
model_name="minioconfiguration",
name="public_base_url",
field=models.URLField(
blank=True,
max_length=500,
null=True,
verbose_name="Public base URL",
),
),
migrations.AddField(
model_name="minioconfiguration",
name="write_prefix",
field=models.CharField(
blank=True,
max_length=128,
null=True,
verbose_name="Write prefix",
),
),
migrations.AlterField(
model_name="filelocation",
name="uri",
field=models.URLField(blank=True, max_length=500, null=True, verbose_name="URI"),
),
]
34 changes: 30 additions & 4 deletions files_storage/minio.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@
import json
import logging
import os
import posixpath
from mimetypes import types_map
from tempfile import NamedTemporaryFile, TemporaryDirectory
from urllib.parse import urljoin

from minio import Minio
from minio.error import S3Error
Expand Down Expand Up @@ -69,9 +71,12 @@ def __init__(
minio_access_key,
minio_secret_key,
bucket_root,
location,
location=None,
minio_secure=True,
minio_http_client=None,
write_prefix="",
public_base_url="",
bucket_subdir=None,
):
self.bucket_root = bucket_root
self.POLICY_READ_ONLY = {
Expand All @@ -97,7 +102,22 @@ def __init__(
self.minio_secure = minio_secure
self.http_client = minio_http_client
self._client_instance = None
self.location = location
self.location = location or bucket_subdir

@robertatakenaka robertatakenaka Jun 29, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rondinelisaad é um erro considerar location como um subdir.

Cria um bucket no MinIO/S3.

self._client — cliente MinIO (instância de Minio).
make_bucket(...) — cria um novo bucket no armazenamento de objetos.
self.bucket_root — nome do bucket a ser criado.
location=self.location — região onde o bucket será criado (ex.: "us-east-1").

Não. make_bucket não aceita caminhos com / — o nome do bucket é plano, não hierárquico.

Buckets S3/MinIO não têm subdiretórios reais. A "estrutura de pastas" é uma ilusão criada por prefixos na chave do objeto, não no nome do bucket.
"pasta/subpasta" como nome de bucket falharia: / é caractere inválido em nome de bucket (regras S3). Você receberia um erro de validação (InvalidBucketName).

# bucket plano
self._client.make_bucket("scielo-assets")

# "pastas" via prefixo na chave do objeto
self._client.put_object(
    "scielo-assets",
    "pasta/subpasta/arquivo.pdf",  # aqui o / é permitido e simula diretórios
    data, length,
)

self.write_prefix = self._normalize_path(write_prefix)
self.public_base_url = (public_base_url or "").strip()

def _normalize_path(self, path):
return (path or "").strip("/")

def _object_name_for_storage(self, object_name):
object_name = self._normalize_path(object_name)
if not self.write_prefix:
return object_name
return posixpath.join(self.write_prefix, object_name)

def _public_uri(self, object_name):
object_name = self._normalize_path(object_name)
return urljoin(f"{self.public_base_url.rstrip('/')}/", object_name)

@property
def _client(self):
Expand Down Expand Up @@ -173,7 +193,12 @@ def get_uri(self, object_name: str) -> str:
MinioStorageGetUriError
"""
try:
url = self._client.presigned_get_object(self.bucket_root, object_name)
if self.public_base_url:
return self._public_uri(object_name)

url = self._client.presigned_get_object(
self.bucket_root, self._object_name_for_storage(object_name)
)
return url.split("?")[0]
except Exception as e:
raise MinioStorageGetUriError(
Expand Down Expand Up @@ -257,10 +282,11 @@ def _fput_object(self, file_path, object_name, mimetype) -> str:
------
MinioStorageNoSuchBucketError
"""
storage_object_name = self._object_name_for_storage(object_name)
try:
self._client.fput_object(
self.bucket_root,
object_name=object_name,
object_name=storage_object_name,
file_path=file_path,
content_type=mimetype,
)
Expand Down
16 changes: 15 additions & 1 deletion files_storage/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,12 @@ class MinioConfiguration(CommonControlField):
bucket_root = models.CharField(
_("Bucket root"), max_length=32, null=True, blank=True
)
write_prefix = models.CharField(
_("Write prefix"), max_length=128, null=True, blank=True
)
public_base_url = models.URLField(
_("Public base URL"), max_length=500, null=True, blank=True
)
location = models.CharField(
_("Location"),
max_length=16,
Expand All @@ -62,6 +68,8 @@ class Meta:
FieldPanel("name"),
FieldPanel("host"),
FieldPanel("bucket_root"),
FieldPanel("write_prefix"),
FieldPanel("public_base_url"),
# FieldPanel("location"),
FieldPanel("access_key"),
FieldPanel("secret_key"),
Expand All @@ -85,6 +93,8 @@ def get_or_create(
secret_key=None,
secure=None,
bucket_root=None,
write_prefix=None,
public_base_url=None,
location=None,
user=None,
):
Expand All @@ -98,6 +108,8 @@ def get_or_create(
files_storage.access_key = access_key
files_storage.secret_key = secret_key
files_storage.bucket_root = bucket_root
files_storage.write_prefix = write_prefix
files_storage.public_base_url = public_base_url
files_storage.location = location
files_storage.creator = user
files_storage.save()
Expand Down Expand Up @@ -125,12 +137,14 @@ def get_files_storage(cls, name, minio_http_client=None):
location=obj.location,
minio_secure=obj.secure,
minio_http_client=minio_http_client,
write_prefix=obj.write_prefix,
public_base_url=obj.public_base_url,
)


class FileLocation(CommonControlField):
basename = models.CharField(_("Basename"), max_length=100, null=True, blank=True)
uri = models.URLField(_("URI"), null=True, blank=True, max_length=200)
uri = models.URLField(_("URI"), null=True, blank=True, max_length=500)

autocomplete_search_field = "uri"

Expand Down
56 changes: 49 additions & 7 deletions files_storage/test_minio.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Create your tests here.
import json
from unittest.mock import Mock, patch
from unittest.mock import ANY, patch

from django.test import TestCase
from minio.error import S3Error
Expand Down Expand Up @@ -90,6 +90,21 @@ def test_get_uri(self, mock_presigned_get_object):
"app_name",
)

def test_get_uri_with_public_base_url(self):
minio_storage = MinioStorage(
minio_host="s3.wasabisys.com",
minio_access_key="minio_access_key",
minio_secret_key="minio_secret_key",
bucket_root="scielo",
location="sa-east-1",
write_prefix="upload",
public_base_url="https://minio.scielo.br/upload",
)

uri = minio_storage.get_uri("journal/article.xml")

self.assertEqual("https://minio.scielo.br/upload/journal/article.xml", uri)

@patch("files_storage.minio.MinioStorage.fput")
def test_register(self, mock_fput):
metadata = self.minio_storage.register(
Expand Down Expand Up @@ -120,6 +135,35 @@ def test__fput_object(self, mock_client_fput_object, mock_get_uri):
"subdir1/subdir2/filename.xml",
)

@patch("files_storage.minio.MinioStorage.get_uri")
@patch("files_storage.minio.Minio.fput_object")
def test__fput_object_uses_write_prefix(self, mock_client_fput_object, mock_get_uri):
minio_storage = MinioStorage(
minio_host="s3.wasabisys.com",
minio_access_key="minio_access_key",
minio_secret_key="minio_secret_key",
bucket_root="scielo",
location="sa-east-1",
write_prefix="upload",
public_base_url="https://minio.scielo.br/upload",
)

minio_storage._fput_object(
"/root/folder1/folder2/filename.xml",
"subdir1/subdir2/filename.xml",
mimetype="mimetype_informado",
)

mock_client_fput_object.assert_called_with(
"scielo",
object_name="upload/subdir1/subdir2/filename.xml",
file_path="/root/folder1/folder2/filename.xml",
content_type="mimetype_informado",
)
mock_get_uri.assert_called_with(
"subdir1/subdir2/filename.xml",
)

@patch("files_storage.minio.MinioStorage.get_uri")
@patch("files_storage.minio.Minio.fput_object")
def test__fput_object_raises_exception(
Expand Down Expand Up @@ -212,25 +256,23 @@ def test_remove(self, mock_remove):
def test_fput_content(self, mock_fput_object):
mock_fput_object.return_value = "uri"
uri = self.minio_storage.fput_content(
content="<article/>",
content=b"<article/>",
mimetype="text/xml",
object_name="object_name",
)
self.assertEqual("uri", uri)

@patch("files_storage.minio.MinioStorage._create_tmp_file")
@patch("files_storage.minio.MinioStorage._fput_object")
def test_fput_content_calls_fput_object(
self, mock_fput_object, mock_create_tmp_file
self, mock_fput_object
):
mock_create_tmp_file.return_value = "/tmp/file.xml"
uri = self.minio_storage.fput_content(
content="<article/>",
content=b"<article/>",
mimetype="text/xml",
object_name="object_name",
)
mock_fput_object.assert_called_with(
file_path="/tmp/file.xml",
file_path=ANY,
object_name="object_name",
mimetype="text/xml",
)
5 changes: 4 additions & 1 deletion files_storage/wagtail_hooks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,13 @@ class MinioConfigurationViewSet(CommonControlFieldViewSet):
"name",
"host",
"bucket_root",
"write_prefix",
"public_base_url",
)
search_fields = (
"name",
"host",
"bucket_root",
"write_prefix",
"public_base_url",
)