From 10e0a396a71267e6a3923eb76299e519aa04beb5 Mon Sep 17 00:00:00 2001 From: hallelx2 Date: Sat, 4 Jul 2026 04:00:20 +0100 Subject: [PATCH 1/2] @ Sanitize document titles + drop repeated icon-glyph marker rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Title sanitize: an ingest title with invalid UTF-8 bytes (e.g. a client that mangled an em-dash in the multipart field) made the Postgres insert fail with "db write failed" and 500 the whole upload. sanitizeTitle now coerces the title to valid UTF-8 and strips control chars before the DB write; the parsed-title path is likewise run through cleanForLLM. A bad title byte can no longer fail an ingest. Cover-page furniture: isRepeatedMarkerLine drops rows that are mostly the same short token repeated — the artifact of an icon glyph rendered as text and stamped once per item (e.g. the ORCID "iD" logo on a paper byline, "iD iD iD iD"), which the font heuristic otherwise surfaced as junk author-block sections. Real prose never repeats an identical <=2-char token as >=60% of a >=3-token line, so this is safe. @ --- internal/handler/documents.go | 36 +++++++++++++++++++++++++ internal/handler/sanitize_title_test.go | 20 ++++++++++++++ pkg/ingest/ingest.go | 4 ++- pkg/parser/pdf.go | 34 +++++++++++++++++++++-- pkg/parser/repeated_marker_test.go | 32 ++++++++++++++++++++++ 5 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 internal/handler/sanitize_title_test.go create mode 100644 pkg/parser/repeated_marker_test.go diff --git a/internal/handler/documents.go b/internal/handler/documents.go index 07d48c8..fa2c661 100644 --- a/internal/handler/documents.go +++ b/internal/handler/documents.go @@ -7,9 +7,11 @@ import ( "io" "log/slog" "net/http" + "regexp" "strconv" "strings" "time" + "unicode/utf8" "github.com/go-chi/chi/v5" @@ -128,6 +130,34 @@ func (h *DocumentsHandler) HandleListDocuments(w http.ResponseWriter, r *http.Re writeJSON(w, http.StatusOK, resp) } +// sanitizeTitle coerces a document title into a value safe for the +// Postgres text column: valid UTF-8, no C0 control chars (except none — +// titles are single-line), whitespace-collapsed and trimmed. Invalid +// byte sequences (a client that mangled a non-ASCII char in the multipart +// field) are dropped rather than replaced so the title stays clean. An +// all-garbage title collapses to "" and the caller falls back to the doc id. +func sanitizeTitle(s string) string { + if !utf8.ValidString(s) { + s = strings.ToValidUTF8(s, "") + } + var b strings.Builder + b.Grow(len(s)) + for _, r := range s { + if r == utf8.RuneError { + continue + } + if r < 0x20 { // drop control chars incl. tab/newline — titles are one line + b.WriteRune(' ') + continue + } + b.WriteRune(r) + } + return strings.TrimSpace(multiSpaceRe.ReplaceAllString(b.String(), " ")) +} + +// multiSpaceRe collapses runs of whitespace in a sanitized title. +var multiSpaceRe = regexp.MustCompile(`\s{2,}`) + // sourceTypeFromContentType collapses an HTTP Content-Type to the // short tag the dashboard's source-type badge expects. func sourceTypeFromContentType(ct string) string { @@ -249,6 +279,12 @@ func (h *DocumentsHandler) HandleIngestDocument(w http.ResponseWriter, r *http.R if title == "" { title = filename } + // Sanitize before the DB write: a title with invalid UTF-8 bytes + // (e.g. a client that mangled an em-dash) would otherwise make the + // Postgres insert fail with "db write failed" and 500 the whole + // ingest. Coerce to valid UTF-8 and drop control chars so a bad + // title byte can never fail the upload. + title = sanitizeTitle(title) if title == "" { title = string(docID) } diff --git a/internal/handler/sanitize_title_test.go b/internal/handler/sanitize_title_test.go new file mode 100644 index 0000000..7f8348d --- /dev/null +++ b/internal/handler/sanitize_title_test.go @@ -0,0 +1,20 @@ +package handler + +import "testing" + +func TestSanitizeTitle(t *testing.T) { + cases := []struct{ in, want string }{ + {"Attention Is All You Need", "Attention Is All You Need"}, + {" spaced out title ", "spaced out title"}, + {"Berkshire — 2023", "Berkshire — 2023"}, // valid UTF-8 em-dash is kept + {"bad\xff\xfebyte", "badbyte"}, // invalid UTF-8 bytes dropped + {"line\nbreak\ttab", "line break tab"}, // control chars → space, collapsed + {"\xff\xfe", ""}, // all-garbage collapses to empty + {"", ""}, + } + for _, c := range cases { + if got := sanitizeTitle(c.in); got != c.want { + t.Errorf("sanitizeTitle(%q) = %q, want %q", c.in, got, c.want) + } + } +} diff --git a/pkg/ingest/ingest.go b/pkg/ingest/ingest.go index f4ab4f1..beb92e7 100644 --- a/pkg/ingest/ingest.go +++ b/pkg/ingest/ingest.go @@ -714,7 +714,9 @@ func (p *Pipeline) persistTree(ctx context.Context, store docPersister, docID tr // Watermarked PDFs whose overlay text shares a Y coordinate with // the real title produce mojibake like "GGlloobbaall SSttrraatteeggyy" // — we'd rather keep the original filename than show that to a user. - if err := store.SetDocumentTitle(ctx, docID, doc.Title); err != nil { + // cleanForLLM strips any invalid-UTF-8 bytes so the title write can + // never fail the persist. + if err := store.SetDocumentTitle(ctx, docID, cleanForLLM(doc.Title)); err != nil { return err } } diff --git a/pkg/parser/pdf.go b/pkg/parser/pdf.go index 9cc961f..19962b9 100644 --- a/pkg/parser/pdf.go +++ b/pkg/parser/pdf.go @@ -1477,12 +1477,42 @@ var boilerplateFragments = []string{ "or scholarly", } +// isRepeatedMarkerLine reports whether a row is a run of the same short +// token repeated — the artifact of an icon glyph rendered as text and +// stamped once per item, e.g. the ORCID "iD" logo repeated once per author +// on a paper's byline ("iD iD iD iD"). Real prose never repeats an +// identical ≤2-character token as the majority of a ≥3-token line, so +// dropping these is safe and removes the junk author-block "headings" +// (e.g. PRISMA's contributor page) the font heuristic would otherwise +// surface as sections. +func isRepeatedMarkerLine(s string) bool { + fields := strings.Fields(s) + if len(fields) < 3 { + return false + } + counts := map[string]int{} + for _, f := range fields { + if len([]rune(f)) <= 2 { + counts[f]++ + } + } + for _, c := range counts { + if c*100 >= len(fields)*60 { // ≥60% of the line is one short token + return true + } + } + return false +} + // isBoilerplateLine reports whether a row is publisher/license noise. // Matches the curated signature list, the bare arXiv id stamp -// ("arXiv:2401.01234v2 [cs.CL] 5 Jan 2024"), and short license-tail -// fragments. +// ("arXiv:2401.01234v2 [cs.CL] 5 Jan 2024"), short license-tail +// fragments, and repeated icon-glyph marker rows (ORCID "iD iD iD"). func isBoilerplateLine(s string) bool { low := strings.ToLower(strings.TrimSpace(s)) + if isRepeatedMarkerLine(low) { + return true + } for _, sig := range boilerplateSignatures { if strings.Contains(low, sig) { return true diff --git a/pkg/parser/repeated_marker_test.go b/pkg/parser/repeated_marker_test.go new file mode 100644 index 0000000..e545982 --- /dev/null +++ b/pkg/parser/repeated_marker_test.go @@ -0,0 +1,32 @@ +package parser + +import "testing" + +// TestIsRepeatedMarkerLine locks the filter that drops icon-glyph marker +// rows (e.g. an ORCID "iD" repeated once per author) while never touching +// real prose or short headings. +func TestIsRepeatedMarkerLine(t *testing.T) { + drop := []string{ + "id id id id", // 4/4 = 100% + "iD iD iD iD", // case-insensitive caller lower-cases first; test raw too + "id id id", // 3/3 + "id id id id author", // 4/5 = 80% ≥ 60% + } + keep := []string{ + "the right to erasure is established in article 17", + "multi-head attention", // 2 tokens, below the 3-token floor + "introduction", // single token + "id id id penny whiting17, david moher 22", // 3 of 9 ≈ 33% → real author line, kept + "id id elie a. akl 8, sue e. brennan", // 2 of 8 → kept + } + for _, s := range drop { + if !isRepeatedMarkerLine(s) { + t.Errorf("expected DROP for %q", s) + } + } + for _, s := range keep { + if isRepeatedMarkerLine(s) { + t.Errorf("expected KEEP for %q", s) + } + } +} From d04cf13f366384a27555d29b20de47ede8cd05d8 Mon Sep 17 00:00:00 2001 From: hallelx2 Date: Sat, 4 Jul 2026 04:12:41 +0100 Subject: [PATCH 2/2] gofmt test files --- internal/handler/sanitize_title_test.go | 6 +++--- pkg/parser/repeated_marker_test.go | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/handler/sanitize_title_test.go b/internal/handler/sanitize_title_test.go index 7f8348d..aa66ac2 100644 --- a/internal/handler/sanitize_title_test.go +++ b/internal/handler/sanitize_title_test.go @@ -7,9 +7,9 @@ func TestSanitizeTitle(t *testing.T) { {"Attention Is All You Need", "Attention Is All You Need"}, {" spaced out title ", "spaced out title"}, {"Berkshire — 2023", "Berkshire — 2023"}, // valid UTF-8 em-dash is kept - {"bad\xff\xfebyte", "badbyte"}, // invalid UTF-8 bytes dropped - {"line\nbreak\ttab", "line break tab"}, // control chars → space, collapsed - {"\xff\xfe", ""}, // all-garbage collapses to empty + {"bad\xff\xfebyte", "badbyte"}, // invalid UTF-8 bytes dropped + {"line\nbreak\ttab", "line break tab"}, // control chars → space, collapsed + {"\xff\xfe", ""}, // all-garbage collapses to empty {"", ""}, } for _, c := range cases { diff --git a/pkg/parser/repeated_marker_test.go b/pkg/parser/repeated_marker_test.go index e545982..e3db729 100644 --- a/pkg/parser/repeated_marker_test.go +++ b/pkg/parser/repeated_marker_test.go @@ -7,15 +7,15 @@ import "testing" // real prose or short headings. func TestIsRepeatedMarkerLine(t *testing.T) { drop := []string{ - "id id id id", // 4/4 = 100% - "iD iD iD iD", // case-insensitive caller lower-cases first; test raw too - "id id id", // 3/3 + "id id id id", // 4/4 = 100% + "iD iD iD iD", // case-insensitive caller lower-cases first; test raw too + "id id id", // 3/3 "id id id id author", // 4/5 = 80% ≥ 60% } keep := []string{ "the right to erasure is established in article 17", - "multi-head attention", // 2 tokens, below the 3-token floor - "introduction", // single token + "multi-head attention", // 2 tokens, below the 3-token floor + "introduction", // single token "id id id penny whiting17, david moher 22", // 3 of 9 ≈ 33% → real author line, kept "id id elie a. akl 8, sue e. brennan", // 2 of 8 → kept }