From c083031b1dd8b79c03f1eb4a79088e1551f24df5 Mon Sep 17 00:00:00 2001 From: hallelx2 Date: Sat, 4 Jul 2026 02:57:08 +0100 Subject: [PATCH] @ Add explicit title override to document ingest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accept an optional "title" on ingest (multipart form field or JSON body). When set it seeds the document row AND is threaded through the ingest Payload so persistTree keeps it — the parsed title never clobbers it. Motivation: some PDFs have unreliable title text the parser latches onto — rotated arXiv margin stamps extracted in reverse ("3202 guA" for "Aug 2023"), repeated ORCID iD markers ("iD iD iD iD"), letter-spaced cover pages. The existing mojibake guard only catches doubled-glyph watermarks. An explicit title lets callers (and curated corpora like the playground demo set) show a clean display name regardless. Empty = keep auto-discovery. @ --- internal/handler/documents.go | 19 +++++++++---- pkg/ingest/ingest.go | 30 ++++++++++++++------ pkg/ingest/persist_content_ref_test.go | 39 +++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 15 deletions(-) diff --git a/internal/handler/documents.go b/internal/handler/documents.go index 80dca58..07d48c8 100644 --- a/internal/handler/documents.go +++ b/internal/handler/documents.go @@ -178,10 +178,11 @@ func (h *DocumentsHandler) HandleIngestDocument(w http.ResponseWriter, r *http.R } var ( - filename string - contentType string - body io.Reader - size int64 + filename string + contentType string + body io.Reader + size int64 + titleOverride string ) ct := r.Header.Get("Content-Type") @@ -201,12 +202,15 @@ func (h *DocumentsHandler) HandleIngestDocument(w http.ResponseWriter, r *http.R contentType = header.Header.Get("Content-Type") body = file size = header.Size + // Optional multipart "title" field overrides the discovered title. + titleOverride = strings.TrimSpace(r.FormValue("title")) case strings.HasPrefix(ct, "application/json"): var payload struct { Filename string `json:"filename"` ContentType string `json:"content_type"` Content string `json:"content"` + Title string `json:"title"` } if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { writeErr(w, http.StatusBadRequest, "invalid json: "+err.Error()) @@ -220,6 +224,7 @@ func (h *DocumentsHandler) HandleIngestDocument(w http.ResponseWriter, r *http.R contentType = payload.ContentType body = strings.NewReader(payload.Content) size = int64(len(payload.Content)) + titleOverride = strings.TrimSpace(payload.Title) default: writeErr(w, http.StatusUnsupportedMediaType, @@ -240,7 +245,10 @@ func (h *DocumentsHandler) HandleIngestDocument(w http.ResponseWriter, r *http.R return } - title := filename + title := titleOverride + if title == "" { + title = filename + } if title == "" { title = string(docID) } @@ -281,6 +289,7 @@ func (h *DocumentsHandler) HandleIngestDocument(w http.ResponseWriter, r *http.R ContentType: contentType, Filename: filename, SourceRef: key, + Title: titleOverride, Profile: r.Header.Get("X-Vectorless-Profile"), }) if err := h.queue.Enqueue(ctx, queue.Job{ diff --git a/pkg/ingest/ingest.go b/pkg/ingest/ingest.go index 4df956c..f4ab4f1 100644 --- a/pkg/ingest/ingest.go +++ b/pkg/ingest/ingest.go @@ -67,6 +67,13 @@ type Payload struct { ContentType string `json:"content_type"` Filename string `json:"filename"` SourceRef string `json:"source_ref"` // storage key of the original bytes + // Title, when set, is an explicit caller-supplied document title that + // overrides whatever the parser discovers from the content. It is + // "sticky": persistTree will NOT overwrite it with the parsed title. + // Useful when the PDF's own title text is unreliable (rotated arXiv + // margin stamps, ORCID iD markers, letter-spaced cover pages) or the + // caller simply wants a curated display name. Empty = auto-discover. + Title string `json:"title,omitempty"` // Profile selects domain-aware structuring/summarization prompts // ("generic", "research", "medical"). Empty = generic. Sourced from // the document's store (the control plane injects X-Vectorless-Profile). @@ -376,7 +383,7 @@ func (p *Pipeline) Run(ctx context.Context, pl Payload) error { } log.Info("ingest: parsed", "sections", len(parsed.Flatten()), "title", parsed.Title) - if err := p.persistTree(ctx, p.DB, pl.DocumentID, parsed); err != nil { + if err := p.persistTree(ctx, p.DB, pl.DocumentID, parsed, pl.Title); err != nil { p.fail(ctx, p.DB, pl.DocumentID, "persist tree", err) return err } @@ -674,7 +681,7 @@ func (p *Pipeline) runMinimal(ctx context.Context, store docPersister, pl Payloa } log.Info("ingest: parsed", "sections", len(parsed.Flatten()), "title", parsed.Title) - if err := p.persistTree(ctx, store, pl.DocumentID, parsed); err != nil { + if err := p.persistTree(ctx, store, pl.DocumentID, parsed, pl.Title); err != nil { p.fail(ctx, store, pl.DocumentID, "persist tree", err) return err } @@ -695,13 +702,18 @@ func (p *Pipeline) runMinimal(ctx context.Context, store docPersister, pl Payloa // The DB operations go through the narrow docPersister interface so the // persist path can be exercised (e.g. by the minimal-mode test) without // a live Postgres; production callers pass p.DB, which satisfies it. -func (p *Pipeline) persistTree(ctx context.Context, store docPersister, docID tree.DocumentID, doc *parser.ParsedDoc) error { - // Only overwrite the row's title (which was seeded with the - // filename at upload time) when the parsed title looks usable. - // Watermarked PDFs whose overlay text shares a Y coordinate with - // the real title produce mojibake like "GGlloobbaall SSttrraatteeggyy" - // — we'd rather keep the original filename than show that to a user. - if doc.Title != "" && !isLikelyMojibakeTitle(doc.Title) { +func (p *Pipeline) persistTree(ctx context.Context, store docPersister, docID tree.DocumentID, doc *parser.ParsedDoc, titleOverride string) error { + // An explicit caller-supplied title is sticky: keep it and never let + // the parsed title clobber it. The row was already seeded with this + // value at upload time, so there is nothing to write here. + if strings.TrimSpace(titleOverride) != "" { + // no-op: the override is already the row's title. + } else if doc.Title != "" && !isLikelyMojibakeTitle(doc.Title) { + // Otherwise only overwrite the row's title (seeded with the + // filename at upload time) when the parsed title looks usable. + // Watermarked PDFs whose overlay text shares a Y coordinate with + // the real title produce mojibake like "GGlloobbaall SSttrraatteeggyy" + // — we'd rather keep the original filename than show that to a user. if err := store.SetDocumentTitle(ctx, docID, doc.Title); err != nil { return err } diff --git a/pkg/ingest/persist_content_ref_test.go b/pkg/ingest/persist_content_ref_test.go index 8627a0a..0496352 100644 --- a/pkg/ingest/persist_content_ref_test.go +++ b/pkg/ingest/persist_content_ref_test.go @@ -9,6 +9,43 @@ import ( "github.com/hallelx2/vectorless-engine/pkg/storage" ) +// TestPersistTree_TitleOverrideIsSticky verifies that an explicit +// caller-supplied title is never clobbered by the parsed title, while a +// blank override still lets a usable parsed title through. +func TestPersistTree_TitleOverrideIsSticky(t *testing.T) { + store, err := storage.NewLocal(t.TempDir()) + if err != nil { + t.Fatalf("NewLocal: %v", err) + } + p := &Pipeline{Storage: store} + doc := &parser.ParsedDoc{ + Title: "Some Parsed Title", + Sections: []parser.Section{ + {Level: 1, Title: "S", Content: "body", PageStart: 1, PageEnd: 1}, + }, + } + + // With an override, persistTree must NOT push the parsed title (the row + // already carries the override from upload time → SetDocumentTitle + // stays uncalled, so the fake's title remains empty). + fake := &fakeDocStore{} + if err := p.persistTree(context.Background(), fake, "doc_x", doc, "Attention Is All You Need"); err != nil { + t.Fatalf("persistTree (override): %v", err) + } + if fake.title != "" { + t.Errorf("override present: parsed title must not overwrite it; SetDocumentTitle called with %q", fake.title) + } + + // With no override, a usable parsed title IS applied. + fake2 := &fakeDocStore{} + if err := p.persistTree(context.Background(), fake2, "doc_y", doc, ""); err != nil { + t.Fatalf("persistTree (no override): %v", err) + } + if fake2.title != "Some Parsed Title" { + t.Errorf("no override: parsed title should apply, got %q", fake2.title) + } +} + // TestPersistTree_ContentRefMatchesStoredObjects is the HAL-316 regression: // a leaf only gets a ContentRef when its content was actually written. An // empty-after-clean leaf must get NO ref (and no stored object), so later @@ -35,7 +72,7 @@ func TestPersistTree_ContentRefMatchesStoredObjects(t *testing.T) { }, } - if err := p.persistTree(context.Background(), fake, "doc_x", doc); err != nil { + if err := p.persistTree(context.Background(), fake, "doc_x", doc, ""); err != nil { t.Fatalf("persistTree: %v", err) }