From 8c4b45547473ab2e74bd91a6e04508ec75c196a2 Mon Sep 17 00:00:00 2001 From: Ben Silverman Date: Tue, 3 Mar 2026 14:40:47 -0500 Subject: [PATCH 1/3] Add data exports (#20) --- manuscript/management/commands/run_exports.py | 42 +++++++++++++ manuscript/resources.py | 59 +++++++++++++++++++ pages/migrations/0008_datapage.py | 34 +++++++++++ pages/models.py | 28 +++++++++ templates/pages/data_page.html | 32 ++++++++++ templates/pages/site_page.html | 17 ++---- 6 files changed, 199 insertions(+), 13 deletions(-) create mode 100644 manuscript/management/commands/run_exports.py create mode 100644 pages/migrations/0008_datapage.py create mode 100644 templates/pages/data_page.html diff --git a/manuscript/management/commands/run_exports.py b/manuscript/management/commands/run_exports.py new file mode 100644 index 0000000..f30ba68 --- /dev/null +++ b/manuscript/management/commands/run_exports.py @@ -0,0 +1,42 @@ +import csv +from django.core.management.base import BaseCommand +from django.core.files.base import ContentFile +from django.core.files.storage import default_storage +from manuscript.resources import ( + LocationResource, + LocationAliasResource, + FolioResource, + SingleManuscriptResource, + StanzaResource, + StanzaTranslatedResource, +) + + +class Command(BaseCommand): + help = "Export records to storage backend for public download" + + def handle(self, *args, **options): + export_targets = [ + (FolioResource(), "exports/folios.csv"), + (SingleManuscriptResource(), "exports/manuscripts.csv"), + (StanzaResource(), "exports/stanzas.csv"), + (StanzaTranslatedResource(), "exports/translated_stanzas.csv"), + (LocationResource(), "exports/toponyms.csv"), + (LocationAliasResource(), "exports/toponym_variants.csv"), + ] + + for resource_class, file_path in export_targets: + self.stdout.write(f"Exporting {file_path}...") + dataset = resource_class.export() + csv_data = dataset.csv + + # since AWS_S3_FILE_OVERWRITE = False, delete the old version first + if default_storage.exists(file_path): + default_storage.delete(file_path) + + # save to default storage + default_storage.save(file_path, ContentFile(csv_data)) + + self.stdout.write( + self.style.SUCCESS("Successfully updated all public exports.") + ) diff --git a/manuscript/resources.py b/manuscript/resources.py index 4d07c14..bf4ba44 100644 --- a/manuscript/resources.py +++ b/manuscript/resources.py @@ -3,6 +3,7 @@ from import_export.results import RowResult from django.contrib import messages from django.db.models import Q +from django.utils.html import strip_tags import logging logger = logging.getLogger(__name__) @@ -16,9 +17,20 @@ Location, LocationAlias, LineCode, + StanzaTranslated, ) +class ExportOnlyResource(resources.ModelResource): + """reusable base class to disable import functionality""" + + def before_import(self, dataset, **kwargs): + raise NotImplementedError("Importing is disabled for this resource.") + + def import_data(self, dataset, **kwargs): + raise NotImplementedError("Importing is disabled for this resource.") + + class FolioResource(resources.ModelResource): """Resource for importing Folio data with proper object logging""" @@ -247,6 +259,9 @@ def dehydrate_mod_name(self, instance): def dehydrate_anc_name(self, instance): return getattr(instance, "Anc_Name", "") + def dehydrate_description(self, instance): + return strip_tags(instance.description) if instance.description else "" + def after_import_row(self, row, row_result, **kwargs): """Create alias records for modern and ancient names if they exist""" # check dry_run to prevent creating LocationAlias records during preview @@ -691,3 +706,47 @@ def import_row(self, row, instance_loader, **kwargs): def get_diff_headers(self): """Define headers for the diff display""" return ["Code", "Toponyms"] + + +class StanzaResource(ExportOnlyResource): + class Meta: + model = Stanza + fields = ( + "id", + "stanza_line_code_starts", + "stanza_line_code_ends", + "stanza_text", + "stanza_notes", + "language", + "is_rubric", + ) + export_order = fields + + def dehydrate_stanza_text(self, instance): + return strip_tags(instance.stanza_text) if instance.stanza_text else "" + + def dehydrate_stanza_notes(self, instance): + return strip_tags(instance.stanza_notes) if instance.stanza_notes else "" + + +class StanzaTranslatedResource(ExportOnlyResource): + # export the line code of the parent stanza instead of just the id + parent_stanza_code = fields.Field( + attribute="stanza__stanza_line_code_starts", column_name="original_stanza_code" + ) + + class Meta: + model = StanzaTranslated + fields = ( + "id", + "parent_stanza_code", + "stanza_line_code_starts", + "stanza_line_code_ends", + "stanza_text", + "language", + "is_rubric", + ) + export_order = fields + + def dehydrate_stanza_text(self, instance): + return strip_tags(instance.stanza_text) if instance.stanza_text else "" diff --git a/pages/migrations/0008_datapage.py b/pages/migrations/0008_datapage.py new file mode 100644 index 0000000..e6cd3d6 --- /dev/null +++ b/pages/migrations/0008_datapage.py @@ -0,0 +1,34 @@ +# Generated by Django 5.1.15 on 2026-02-19 18:40 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("pages", "0007_alter_aboutpage_body_alter_homeintroduction_body_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="DataPage", + fields=[ + ( + "sitepage_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="pages.sitepage", + ), + ), + ], + options={ + "abstract": False, + }, + bases=("pages.sitepage",), + ), + ] diff --git a/pages/models.py b/pages/models.py index ada4270..06f47b3 100644 --- a/pages/models.py +++ b/pages/models.py @@ -3,6 +3,7 @@ from wagtail.fields import RichTextField, StreamField from wagtail.models import Page from wagtail.snippets.models import register_snippet +from django.conf import settings from django.db import models from wagtail import blocks @@ -73,3 +74,30 @@ class ManuscriptsIntroduction(models.Model): def __str__(self): return self.title + + +class DataPage(SitePage): + template = "pages/data_page.html" + + def get_context(self, request): + """add our public export URLs to context to include in the template""" + context = super().get_context(request) + base_url = settings.MEDIA_URL.rstrip("/") + files = [ + ("Manuscripts", "manuscripts"), + ("Folios", "folios"), + ("Stanzas (Italian)", "stanzas"), + ("Stanzas (English)", "translated_stanzas"), + ("Toponyms", "toponyms"), + ("Toponym Variants", "toponym_variants"), + ] + export_list = [] + for label, slug in files: + export_list.append( + { + "label": label, + "csv": f"{base_url}/exports/{slug}.csv", + } + ) + context["exports"] = export_list + return context diff --git a/templates/pages/data_page.html b/templates/pages/data_page.html new file mode 100644 index 0000000..6b3b2ea --- /dev/null +++ b/templates/pages/data_page.html @@ -0,0 +1,32 @@ +{% extends "base.html" %} +{% load wagtailcore_tags %} + +{% block title %}{{ page.title }} - La Sfera{% endblock title %} + +{% block content %} +
+
+

+ {{ page.title }} +

+ +
+ {# Left Column (2/3) #} +
+ {% include_block page.body %} +
+ + {# Right Column (1/3) #} + +
+
+
+{% endblock %} diff --git a/templates/pages/site_page.html b/templates/pages/site_page.html index c9ec63b..73fb1db 100644 --- a/templates/pages/site_page.html +++ b/templates/pages/site_page.html @@ -5,22 +5,13 @@ {% block content %}
-
+

{{ page.title }}

- -
- {# Left Column (2/3) #} -
- {% include_block page.body %} -
- - {# Right Column (1/3) #} - -
+
+ {% include_block page.body %} +
{% endblock %} From 7cb7edd04a88d571f0e4546387477bce43cf7f8f Mon Sep 17 00:00:00 2001 From: Ben Silverman Date: Tue, 3 Mar 2026 14:58:28 -0500 Subject: [PATCH 2/3] env improvements --- .env.example | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/.env.example b/.env.example index 2ecbf67..04157f6 100644 --- a/.env.example +++ b/.env.example @@ -1,24 +1,27 @@ -# Set True for Django debug mode +# Django Core DEBUG=True -# Set True to enable verbose SQL query logging SQL_DEBUG=False DJANGO_SECRET_KEY=thisisnotasecretkey -DJANGO_ALLOWED_HOSTS='localhost,"",127.0.0.1,0.0.0.0' -DJANGO_CSRF_TRUSTED_ORIGINS='http://localhost' +DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1,0.0.0.0 +DJANGO_CSRF_TRUSTED_ORIGINS=http://localhost:8000,http://127.0.0.1:8000 -# Database +# Database DB_NAME=lasfera DB_USER=lasfera -DB_PASSWORD=postgres +DB_PASS=postgres DB_HOST=db +DB_PORT=5432 +# Only used in Docker setup POSTGRES_DB=lasfera POSTGRES_USER=lasfera POSTGRES_PASSWORD=postgres -POSTGRES_HOST=db -# File storage (only required for production deployments) -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= -AWS_STORAGE_BUCKET_NAME= +# Wagtail +WAGTAILADMIN_BASE_URL=http://localhost:8000 + +# AWS (Leave as placeholders for local dev) +AWS_ACCESS_KEY_ID=local_dev +AWS_SECRET_ACCESS_KEY=local_dev +AWS_STORAGE_BUCKET_NAME=local_dev AWS_S3_REGION_NAME=us-east-1 From 06b34837ace69d058164164c66f6d4d45965707d Mon Sep 17 00:00:00 2001 From: Ben Silverman Date: Tue, 3 Mar 2026 14:58:49 -0500 Subject: [PATCH 3/3] Improved file handling (#20) --- pages/models.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pages/models.py b/pages/models.py index 06f47b3..3005e5b 100644 --- a/pages/models.py +++ b/pages/models.py @@ -4,6 +4,7 @@ from wagtail.models import Page from wagtail.snippets.models import register_snippet from django.conf import settings +from django.core.files.storage import default_storage from django.db import models from wagtail import blocks @@ -82,7 +83,6 @@ class DataPage(SitePage): def get_context(self, request): """add our public export URLs to context to include in the template""" context = super().get_context(request) - base_url = settings.MEDIA_URL.rstrip("/") files = [ ("Manuscripts", "manuscripts"), ("Folios", "folios"), @@ -93,11 +93,14 @@ def get_context(self, request): ] export_list = [] for label, slug in files: - export_list.append( - { + file_path = f"exports/{slug}.csv" + # Use default_storage to generate the safe S3 URL + # Fallback to '#' if the file hasn't been generated by the command yet + if default_storage.exists(file_path): + file_url = default_storage.url(file_path) + export_list.append({ "label": label, - "csv": f"{base_url}/exports/{slug}.csv", - } - ) + "csv": file_url, + }) context["exports"] = export_list return context